00000001:01000000:3.0F:1789760611.841270:0:2280:0:(lnet-crypto.c:360:cfs_crypto_performance_test()) Crypto hash algorithm adler32 speed = 803 MB/s 00000001:01000000:1.0F:1789760612.091009:0:2280:0:(lnet-crypto.c:360:cfs_crypto_performance_test()) Crypto hash algorithm crc32 speed = 1151 MB/s 00000001:01000000:0.0F:1789760612.340901:0:2280:0:(lnet-crypto.c:360:cfs_crypto_performance_test()) Crypto hash algorithm crc32c speed = 9539 MB/s 00000020:02000000:2.0F:1789760613.027011:0:2309:0:(class_obd.c:771:obdclass_init()) Lustre: Build Version: 2.17.57_103_g082f77f 00000400:02000000:3.0:1789760613.995809:0:2324:0:(api-ni.c:2823:lnet_startup_lndni()) Added LNI 192.168.203.7@tcp [8/256/0/180] 00008000:02000000:3.0:1789760618.067719:0:2475:0:(echo_client.c:2574:obdecho_init()) Echo OBD driver; http://www.lustre.org/ 00000020:01000004:0.0:1789760745.043638:0:5415:0:(obd_mount.c:923:lmd_print()) mount data: 00000020:01000004:0.0:1789760745.043645:0:5415:0:(obd_mount.c:925:lmd_print()) profile: lustre-client 00000020:01000004:0.0:1789760745.043646:0:5415:0:(obd_mount.c:926:lmd_print()) device: 192.168.203.107@tcp:/lustre 00000020:01000004:0.0:1789760745.043648:0:5415:0:(obd_mount.c:929:lmd_print()) options: user_xattr,flock 00000080:01200004:0.0:1789760745.043896:0:5415:0:(super25.c:131:lustre_fill_super()) VFS Op: sb 0000000056848a2f 00000080:01000004:0.0:1789760745.043910:0:5415:0:(super25.c:162:lustre_fill_super()) Mounting client lustre-client 00000020:01000004:0.0:1789760745.043930:0:5415:0:(obd_mount.c:411:lustre_start_mgc()) Start MGC 'MGC192.168.203.107@tcp' 00000020:01000004:0.0:1789760745.045255:0:5415:0:(obd_mount.c:230:lustre_start_simple()) Starting OBD MGC192.168.203.107@tcp (typ=mgc) 00000020:00000080:0.0:1789760745.045298:0:5415:0:(genops.c:413:class_newdev()) Allocate new device MGC192.168.203.107@tcp (00000000ec3571cd) 00000020:00000080:0.0:1789760745.045317:0:5415:0:(obd_config.c:720:class_attach_name()) OBD: dev 0 attached type mgc with refcount 2 00000100:00100000:0.0:1789760745.045627:0:5415:0:(service.c:155:ptlrpc_grow_req_bufs()) ldlm_cbd: allocate 1 new 8192-byte reqbufs (0/1 left), rc = 0 00000100:00100000:0.0:1789760745.045654:0:5415:0:(service.c:155:ptlrpc_grow_req_bufs()) ldlm_cbd: allocate 1 new 8192-byte reqbufs (0/1 left), rc = 0 00000100:00100000:0.0:1789760745.045835:0:5415:0:(service.c:3462:ptlrpc_start_thread()) ldlm_cbd[0] started 0 min 2 max 34 00000100:00100000:0.0:1789760745.045842:0:5415:0:(service.c:3521:ptlrpc_start_thread()) starting thread 'ldlm_cb00_000' 00000100:00100000:2.0:1789760745.048879:0:5415:0:(service.c:3462:ptlrpc_start_thread()) ldlm_cbd[0] started 1 min 2 max 34 00000100:00100000:2.0:1789760745.048894:0:5415:0:(service.c:3521:ptlrpc_start_thread()) starting thread 'ldlm_cb00_001' 00000100:00100000:2.0:1789760745.049317:0:5415:0:(service.c:3462:ptlrpc_start_thread()) ldlm_cbd[1] started 0 min 2 max 34 00000100:00100000:2.0:1789760745.049324:0:5415:0:(service.c:3521:ptlrpc_start_thread()) starting thread 'ldlm_cb01_000' 00000100:00100000:3.0:1789760745.050093:0:5415:0:(service.c:3462:ptlrpc_start_thread()) ldlm_cbd[1] started 1 min 2 max 34 00000100:00100000:3.0:1789760745.050114:0:5415:0:(service.c:3521:ptlrpc_start_thread()) starting thread 'ldlm_cb01_001' 00000100:00100000:2.0:1789760745.053190:0:5415:0:(service.c:155:ptlrpc_grow_req_bufs()) ldlm_canceld: allocate 64 new 8192-byte reqbufs (0/64 left), rc = 0 00000100:00100000:2.0:1789760745.053547:0:5415:0:(service.c:155:ptlrpc_grow_req_bufs()) ldlm_canceld: allocate 64 new 8192-byte reqbufs (0/64 left), rc = 0 00000100:00100000:2.0:1789760745.055321:0:5415:0:(service.c:3462:ptlrpc_start_thread()) ldlm_canceld[0] started 0 min 3 max 34 00000100:00100000:2.0:1789760745.055338:0:5415:0:(service.c:3521:ptlrpc_start_thread()) starting thread 'ldlm_cn00_000' 00000100:00100000:2.0:1789760745.060575:0:5415:0:(service.c:3462:ptlrpc_start_thread()) ldlm_canceld[0] started 1 min 3 max 34 00000100:00100000:2.0:1789760745.060596:0:5415:0:(service.c:3521:ptlrpc_start_thread()) starting thread 'ldlm_cn00_001' 00000100:00100000:3.0:1789760745.061718:0:5415:0:(service.c:3462:ptlrpc_start_thread()) ldlm_canceld[0] started 2 min 3 max 34 00000100:00100000:3.0:1789760745.061729:0:5415:0:(service.c:3521:ptlrpc_start_thread()) starting thread 'ldlm_cn00_002' 00000100:00100000:2.0:1789760745.065883:0:5415:0:(service.c:3462:ptlrpc_start_thread()) ldlm_canceld[1] started 0 min 3 max 34 00000100:00100000:2.0:1789760745.065904:0:5415:0:(service.c:3521:ptlrpc_start_thread()) starting thread 'ldlm_cn01_000' 00000100:00100000:3.0:1789760745.069744:0:5415:0:(service.c:3462:ptlrpc_start_thread()) ldlm_canceld[1] started 1 min 3 max 34 00000100:00100000:2.0:1789760745.071251:0:5415:0:(service.c:3521:ptlrpc_start_thread()) starting thread 'ldlm_cn01_001' 00000100:00100000:2.0:1789760745.072028:0:5415:0:(service.c:3462:ptlrpc_start_thread()) ldlm_canceld[1] started 2 min 3 max 34 00000100:00100000:2.0:1789760745.072037:0:5415:0:(service.c:3521:ptlrpc_start_thread()) starting thread 'ldlm_cn01_002' 00010000:00010000:2.0:1789760745.075291:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789760745.075300:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789760745.075303:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789760745.077635:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789760745.077643:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789760745.077646:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00080000:3.0:1789760745.078932:0:5415:0:(ldlm_lib.c:102:import_set_conn()) imp 0000000082f6baaa@MGC192.168.203.107@tcp: add connection 192.168.203.107@tcp at head 00010000:00010000:3.0:1789760745.079990:0:5415:0:(ldlm_pool.c:923:ldlm_pool_init()) Lock pool ldlm-pool-MGC192.168.203.107@tcp-0 is initialized 00000040:01000000:3.0:1789760745.080008:0:5415:0:(llog_obd.c:197:llog_setup()) obd MGC192.168.203.107@tcp ctxt 1 is initialized 00010000:00010000:2.0:1789760745.083043:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 10000000:01000000:0.0:1789760745.093151:0:5431:0:(mgc_request.c:631:mgc_requeue_thread()) Starting requeue thread 00000020:00000080:2.0:1789760745.093254:0:5415:0:(obd_config.c:826:class_setup()) finished setup of obd MGC192.168.203.107@tcp (uuid 02bc4b43-5e6f-4d7e-9224-80f5c789104f) 00000020:00000080:2.0:1789760745.093288:0:5415:0:(genops.c:1385:class_connect()) connect: client 02bc4b43-5e6f-4d7e-9224-80f5c789104f, cookie 0x59bc99bb61948051 00000100:00080000:2.0:1789760745.093308:0:5415:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 0 00000100:00080000:2.0:1789760745.093324:0:5415:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import 0000000082f6baaa using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789760745.093408:0:5415:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000003aeb691d x1876700024537216/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789760745.093441:0:5415:0:(pinger.c:373:ptlrpc_pinger_add_import()) adding pingable import 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS 00000080:00200000:2.0:1789760745.093471:0:5415:0:(llite_lib.c:1506:ll_fill_super()) VFS Op: cfg_instance lustre-client-ffffa06c18a1e000 (sb 0000000056848a2f) 00000080:01000000:2.0:1789760745.093740:0:5415:0:(llite_lib.c:1232:ll_options()) Parsing opts user_xattr,flock 00000080:00000004:2.0:1789760745.093749:0:5415:0:(llite_lib.c:1242:ll_options()) next opt=user_xattr 00000080:00000004:2.0:1789760745.093754:0:5415:0:(llite_lib.c:1242:ll_options()) next opt=flock 00000080:01000000:2.0:1789760745.093762:0:5415:0:(llite_lib.c:1540:ll_fill_super()) llite sb uuid: 734aaee5-e21a-40f9-9d4e-90e3c76738d2 10000000:01000000:2.0:1789760745.094379:0:5415:0:(mgc_request.c:1944:mgc_process_config()) parse_log lustre-client from 0 10000000:01000000:2.0:1789760745.094383:0:5415:0:(mgc_request.c:329:config_log_add()) add config log lustre-client-ffffa06c18a1e000 10000000:01000000:2.0:1789760745.094387:0:5415:0:(mgc_request.c:194:do_config_log_add()) do adding config log lustre-sptlrpc-ffffa06c2011cd28 10000000:01000000:2.0:1789760745.094392:0:5415:0:(mgc_request.c:68:mgc_name2resid()) log lustre-sptlrpc to resid 0x65727473756c/0x0 (lustre) 10000000:01000000:2.0:1789760745.094402:0:5415:0:(mgc_request.c:1802:mgc_process_log()) Process log lustre-sptlrpc-ffffa06c2011cd28 from 1 10000000:01000000:2.0:1789760745.094405:0:5415:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00010000:00010000:2.0:1789760745.094445:0:5415:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.203.107@tcp lock: 00000000ab5ede15/0x59bc99bb61948058 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 5415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1789760745.094457:0:5415:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.203.107@tcp lock: 00000000ab5ede15/0x59bc99bb61948058 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 5415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1789760745.094472:0:5415:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.203.107@tcp lock: 00000000ab5ede15/0x59bc99bb61948058 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 5415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00080000:2.0:1789760745.098646:0:5415:0:(client.c:1845:ptlrpc_send_new_req()) @@@ req waiting for recovery: (FULL != CONNECTING) req@000000005fa644d7 x1876700024537344/t0(0) o101->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Rpc:WQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789760745.098670:0:5415:0:(client.c:2715:ptlrpc_set_wait()) set 00000000cffffba4 going to sleep for 0 seconds 00000100:00100000:1.0:1789760745.102363:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@000000003aeb691d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024537216:192.168.203.107@tcp:250: 00000100:00100000:3.0:1789760745.113642:0:2340:0:(client.c:3125:ptlrpc_free_committed()) MGC192.168.203.107@tcp: committing for last_committed 0 gen 1 00000100:00080000:3.0:1789760745.113662:0:2340:0:(import.c:1132:ptlrpc_connect_interpret()) MGC192.168.203.107@tcp: connect to target with instance 0 00000100:00080000:3.0:1789760745.113674:0:2340:0:(import.c:956:ptlrpc_connect_set_flags()) MGC192.168.203.107@tcp: Resetting ns_connect_flags to server flags: 0xa000011001002060 10000000:01000000:3.0:1789760745.113680:0:2340:0:(mgc_request.c:1168:mgc_import_event()) import event 0x808005 00000100:00080000:3.0:1789760745.113684:0:2340:0:(import.c:65:import_set_state_nolock()) 0000000082f6baaa MGS: changing import state from CONNECTING to FULL 10000000:01000000:3.0:1789760745.113690:0:2340:0:(mgc_request.c:1168:mgc_import_event()) import event 0x808004 00000100:00080000:3.0:1789760745.113717:0:2340:0:(pinger.c:186:ptlrpc_pinger_ir_up()) IR up 00000100:00080000:3.0:1789760745.113730:0:2340:0:(recover.c:191:ptlrpc_wake_delayed()) @@@ waking (set 00000000cffffba4): req@000000005fa644d7 x1876700024537344/t0(0) o101->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Rpc:WQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789760745.113754:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@000000003aeb691d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024537216:192.168.203.107@tcp:250: 00000100:00100000:3.0:1789760745.114433:0:5415:0:(client.c:2715:ptlrpc_set_wait()) set 00000000cffffba4 going to sleep for 11 seconds 00000100:00100000:3.0:1789760745.128232:0:5415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@000000005fa644d7 pname:cluuid:pid:xid:nid:opc:job mount.lustre:02bc4b43-5e6f-4d7e-9224-80f5c789104f:5415:1876700024537344:192.168.203.107@tcp:101: 00010000:00010000:3.0:1789760745.128274:0:5415:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: MGC192.168.203.107@tcp lock: 00000000ab5ede15/0x59bc99bb61948058 lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x1daf3002fe9e0860 expref: -99 pid: 5415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1789760745.128291:0:5415:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: MGC192.168.203.107@tcp lock: 00000000ab5ede15/0x59bc99bb61948058 lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x1daf3002fe9e0860 expref: -99 pid: 5415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1789760745.128644:0:5415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@00000000e52c9482 pname:cluuid:pid:xid:nid:opc:job mount.lustre:02bc4b43-5e6f-4d7e-9224-80f5c789104f:5415:1876700024537472:192.168.203.107@tcp:501: 00000100:00100000:2.0:1789760745.128954:0:5415:0:(client.c:2715:ptlrpc_set_wait()) set 000000001fbea7b0 going to sleep for 11 seconds 00000100:00100000:2.0:1789760745.139008:0:5415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@00000000e52c9482 pname:cluuid:pid:xid:nid:opc:job mount.lustre:02bc4b43-5e6f-4d7e-9224-80f5c789104f:5415:1876700024537472:192.168.203.107@tcp:501: 10000000:01000000:2.0:1789760745.139072:0:5415:0:(mgc_request.c:1879:mgc_process_log()) MGC192.168.203.107@tcp: configuration from log 'lustre-sptlrpc' failed (-2). 00010000:00010000:2.0:1789760745.139087:0:5415:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.203.107@tcp lock: 00000000ab5ede15/0x59bc99bb61948058 lrc: 3/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x1daf3002fe9e0860 expref: -99 pid: 5415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1789760745.139112:0:5415:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: MGC192.168.203.107@tcp lock: 00000000ab5ede15/0x59bc99bb61948058 lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x1daf3002fe9e0860 expref: -99 pid: 5415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:2.0:1789760745.139135:0:5415:0:(mgc_request.c:194:do_config_log_add()) do adding config log params-ffffa06c18a1e000 10000000:01000000:2.0:1789760745.139141:0:5415:0:(mgc_request.c:68:mgc_name2resid()) log params to resid 0x736d61726170/0x3 (params) 10000000:01000000:2.0:1789760745.139146:0:5415:0:(mgc_request.c:194:do_config_log_add()) do adding config log lustre-client-ffffa06c18a1e000 10000000:01000000:2.0:1789760745.139148:0:5415:0:(mgc_request.c:68:mgc_name2resid()) log lustre-client to resid 0x65727473756c/0x0 (lustre) 10000000:01000000:2.0:1789760745.139152:0:5415:0:(mgc_request.c:194:do_config_log_add()) do adding config log lustre-cliir-ffffa06c18a1e000 10000000:01000000:2.0:1789760745.139154:0:5415:0:(mgc_request.c:68:mgc_name2resid()) log lustre-cliir to resid 0x65727473756c/0x2 (lustre) 10000000:01000000:2.0:1789760745.139158:0:5415:0:(mgc_request.c:1802:mgc_process_log()) Process log lustre-client-ffffa06c18a1e000 from 1 10000000:01000000:2.0:1789760745.139160:0:5415:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-client (res 0x65727473756c) 00010000:00010000:2.0:1789760745.139198:0:5415:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.203.107@tcp lock: 000000003a95d4a8/0x59bc99bb6194805f lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 5415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1789760745.139207:0:5415:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.203.107@tcp lock: 000000003a95d4a8/0x59bc99bb6194805f lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 5415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1789760745.139217:0:5415:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.203.107@tcp lock: 000000003a95d4a8/0x59bc99bb6194805f lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 5415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1789760745.139238:0:5415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@00000000e52c9482 pname:cluuid:pid:xid:nid:opc:job mount.lustre:02bc4b43-5e6f-4d7e-9224-80f5c789104f:5415:1876700024537600:192.168.203.107@tcp:101: 00000100:00100000:2.0:1789760745.139338:0:5415:0:(client.c:2715:ptlrpc_set_wait()) set 000000001fbea7b0 going to sleep for 11 seconds 00000100:00100000:3.0:1789760745.144559:0:5415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@00000000e52c9482 pname:cluuid:pid:xid:nid:opc:job mount.lustre:02bc4b43-5e6f-4d7e-9224-80f5c789104f:5415:1876700024537600:192.168.203.107@tcp:101: 00010000:00010000:3.0:1789760745.144614:0:5415:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: MGC192.168.203.107@tcp lock: 000000003a95d4a8/0x59bc99bb6194805f lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x11000000000000 nid: local remote: 0x1daf3002fe9e0867 expref: -99 pid: 5415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1789760745.144639:0:5415:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: MGC192.168.203.107@tcp lock: 000000003a95d4a8/0x59bc99bb6194805f lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x1000000000000 nid: local remote: 0x1daf3002fe9e0867 expref: -99 pid: 5415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1789760745.144748:0:5415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@000000002517a04c pname:cluuid:pid:xid:nid:opc:job mount.lustre:02bc4b43-5e6f-4d7e-9224-80f5c789104f:5415:1876700024537728:192.168.203.107@tcp:501: 00000100:00100000:3.0:1789760745.144866:0:5415:0:(client.c:2715:ptlrpc_set_wait()) set 00000000bccdb38d going to sleep for 11 seconds 00000100:00100000:3.0:1789760745.147053:0:5415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@000000002517a04c pname:cluuid:pid:xid:nid:opc:job mount.lustre:02bc4b43-5e6f-4d7e-9224-80f5c789104f:5415:1876700024537728:192.168.203.107@tcp:501: 00000100:00100000:3.0:1789760745.147137:0:5415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@000000002517a04c pname:cluuid:pid:xid:nid:opc:job mount.lustre:02bc4b43-5e6f-4d7e-9224-80f5c789104f:5415:1876700024537856:192.168.203.107@tcp:503: 00000100:00100000:3.0:1789760745.147211:0:5415:0:(client.c:2715:ptlrpc_set_wait()) set 00000000bccdb38d going to sleep for 11 seconds 00000100:00100000:3.0:1789760745.150829:0:5415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@000000002517a04c pname:cluuid:pid:xid:nid:opc:job mount.lustre:02bc4b43-5e6f-4d7e-9224-80f5c789104f:5415:1876700024537856:192.168.203.107@tcp:503: 00000100:00100000:0.0:1789760745.152175:0:5435:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@00000000bdd34edf pname:cluuid:pid:xid:nid:opc:job llog_process_th:02bc4b43-5e6f-4d7e-9224-80f5c789104f:5435:1876700024537984:192.168.203.107@tcp:502: 00000100:00100000:0.0:1789760745.152493:0:5435:0:(client.c:2715:ptlrpc_set_wait()) set 0000000027079f3c going to sleep for 11 seconds 00000100:00100000:2.0:1789760745.159222:0:5435:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@00000000bdd34edf pname:cluuid:pid:xid:nid:opc:job llog_process_th:02bc4b43-5e6f-4d7e-9224-80f5c789104f:5435:1876700024537984:192.168.203.107@tcp:502: 00000020:01000000:2.0:1789760745.160354:0:5435:0:(obd_config.c:1893:class_config_llog_handler()) Marker, inst_flg=0x0 mark_flg=0x1 00000020:00000080:2.0:1789760745.160374:0:5435:0:(obd_config.c:1472:class_process_config()) processing cmd: cf010 00000020:00000080:2.0:1789760745.160377:0:5435:0:(obd_config.c:1563:class_process_config()) marker 4 (0x1) lustre-clilov lov setup 00000020:01000000:2.0:1789760745.160387:0:5435:0:(obd_config.c:2004:class_config_llog_handler()) cmd cf001, instance name: lustre-clilov-ffffa06c18a1e000 00000020:00000080:2.0:1789760745.160390:0:5435:0:(obd_config.c:1472:class_process_config()) processing cmd: cf001 00000020:00000080:2.0:1789760745.160426:0:5435:0:(genops.c:413:class_newdev()) Allocate new device lustre-clilov-ffffa06c18a1e000 (000000000cb4504e) 00000020:00000080:2.0:1789760745.160561:0:5435:0:(obd_config.c:720:class_attach_name()) OBD: dev 1 attached type lov with refcount 2 00000020:01000000:2.0:1789760745.160569:0:5435:0:(obd_config.c:2004:class_config_llog_handler()) cmd cf003, instance name: lustre-clilov-ffffa06c18a1e000 00000020:00000080:2.0:1789760745.160579:0:5435:0:(obd_config.c:1472:class_process_config()) processing cmd: cf003 00000020:00000080:2.0:1789760745.161902:0:5435:0:(obd_config.c:826:class_setup()) finished setup of obd lustre-clilov-ffffa06c18a1e000 (uuid 734aaee5-e21a-40f9-9d4e-90e3c76738d2) 00000020:01000000:2.0:1789760745.161919:0:5435:0:(obd_config.c:1893:class_config_llog_handler()) Marker, inst_flg=0x2 mark_flg=0x2 00000020:00000080:2.0:1789760745.161926:0:5435:0:(obd_config.c:1472:class_process_config()) processing cmd: cf010 00000020:00000080:2.0:1789760745.161928:0:5435:0:(obd_config.c:1563:class_process_config()) marker 4 (0x2) lustre-clilov lov setup 00000020:01000000:2.0:1789760745.161932:0:5435:0:(obd_config.c:1893:class_config_llog_handler()) Marker, inst_flg=0x0 mark_flg=0x1 00000020:00000080:2.0:1789760745.161937:0:5435:0:(obd_config.c:1472:class_process_config()) processing cmd: cf010 00000020:00000080:2.0:1789760745.161939:0:5435:0:(obd_config.c:1563:class_process_config()) marker 5 (0x1) lustre-clilmv lmv setup 00000020:01000000:2.0:1789760745.161946:0:5435:0:(obd_config.c:2004:class_config_llog_handler()) cmd cf001, instance name: lustre-clilmv-ffffa06c18a1e000 00000020:00000080:2.0:1789760745.161950:0:5435:0:(obd_config.c:1472:class_process_config()) processing cmd: cf001 00000020:00000080:2.0:1789760745.161963:0:5435:0:(genops.c:413:class_newdev()) Allocate new device lustre-clilmv-ffffa06c18a1e000 (0000000045afc170) 00000020:00000080:2.0:1789760745.161985:0:5435:0:(obd_config.c:720:class_attach_name()) OBD: dev 2 attached type lmv with refcount 2 00000020:01000000:2.0:1789760745.161991:0:5435:0:(obd_config.c:2004:class_config_llog_handler()) cmd cf003, instance name: lustre-clilmv-ffffa06c18a1e000 00000020:00000080:2.0:1789760745.161995:0:5435:0:(obd_config.c:1472:class_process_config()) processing cmd: cf003 00000020:00000080:2.0:1789760745.168077:0:5435:0:(obd_config.c:826:class_setup()) finished setup of obd lustre-clilmv-ffffa06c18a1e000 (uuid 734aaee5-e21a-40f9-9d4e-90e3c76738d2) 00000020:01000000:2.0:1789760745.168106:0:5435:0:(obd_config.c:1893:class_config_llog_handler()) Marker, inst_flg=0x2 mark_flg=0x2 00000020:00000080:2.0:1789760745.168115:0:5435:0:(obd_config.c:1472:class_process_config()) processing cmd: cf010 00000020:00000080:2.0:1789760745.168118:0:5435:0:(obd_config.c:1563:class_process_config()) marker 5 (0x2) lustre-clilmv lmv setup 00000020:01000000:2.0:1789760745.168122:0:5435:0:(obd_config.c:1893:class_config_llog_handler()) Marker, inst_flg=0x0 mark_flg=0x1 00000020:00000080:2.0:1789760745.168128:0:5435:0:(obd_config.c:1472:class_process_config()) processing cmd: cf010 00000020:00000080:2.0:1789760745.168130:0:5435:0:(obd_config.c:1563:class_process_config()) marker 6 (0x1) lustre-client mount opts 00000020:00000080:2.0:1789760745.168134:0:5435:0:(obd_config.c:1472:class_process_config()) processing cmd: cf007 00000020:00000080:2.0:1789760745.168136:0:5435:0:(obd_config.c:1516:class_process_config()) mountopt: profile lustre-client osc lustre-clilov mdc lustre-clilmv 00000020:01000000:2.0:1789760745.168139:0:5435:0:(obd_config.c:1149:class_add_profile()) Add profile lustre-client 00000020:01000000:2.0:1789760745.168145:0:5435:0:(obd_config.c:1893:class_config_llog_handler()) Marker, inst_flg=0x2 mark_flg=0x2 00000020:00000080:2.0:1789760745.168147:0:5435:0:(obd_config.c:1472:class_process_config()) processing cmd: cf010 00000020:00000080:2.0:1789760745.168148:0:5435:0:(obd_config.c:1563:class_process_config()) marker 6 (0x2) lustre-client mount opts 00000020:01000000:2.0:1789760745.168150:0:5435:0:(obd_config.c:1893:class_config_llog_handler()) Marker, inst_flg=0x0 mark_flg=0x1 00000020:00000080:2.0:1789760745.168154:0:5435:0:(obd_config.c:1472:class_process_config()) processing cmd: cf010 00000020:00000080:2.0:1789760745.168155:0:5435:0:(obd_config.c:1563:class_process_config()) marker 7 (0x1) lustre-MDT0000 add mdc 00000020:00000080:2.0:1789760745.168158:0:5435:0:(obd_config.c:1472:class_process_config()) processing cmd: cf005 00000020:00000080:2.0:1789760745.168164:0:5435:0:(obd_config.c:1487:class_process_config()) adding mapping from uuid 192.168.203.107@tcp to nid 0x20000c0a8cb6b (192.168.203.107@tcp) 00000020:01000000:2.0:1789760745.168197:0:5435:0:(obd_config.c:2004:class_config_llog_handler()) cmd cf001, instance name: lustre-MDT0000-mdc-ffffa06c18a1e000 00000020:00000080:2.0:1789760745.168207:0:5435:0:(obd_config.c:1472:class_process_config()) processing cmd: cf001 00000020:00000080:2.0:1789760745.168221:0:5435:0:(genops.c:413:class_newdev()) Allocate new device lustre-MDT0000-mdc-ffffa06c18a1e000 (00000000541f36f2) 00000020:00000080:2.0:1789760745.168244:0:5435:0:(obd_config.c:720:class_attach_name()) OBD: dev 3 attached type mdc with refcount 2 00000020:01000000:2.0:1789760745.168251:0:5435:0:(obd_config.c:2004:class_config_llog_handler()) cmd cf003, instance name: lustre-MDT0000-mdc-ffffa06c18a1e000 00000020:00000080:2.0:1789760745.168254:0:5435:0:(obd_config.c:1472:class_process_config()) processing cmd: cf003 00010000:00080000:2.0:1789760745.168370:0:5435:0:(ldlm_lib.c:102:import_set_conn()) imp 0000000087d4bf82@lustre-MDT0000-mdc-ffffa06c18a1e000: add connection 192.168.203.107@tcp at head 00010000:00010000:2.0:1789760745.177629:0:5435:0:(ldlm_pool.c:923:ldlm_pool_init()) Lock pool ldlm-pool-lustre-MDT0000-mdc-ffffa06c18a1e000-1 is initialized 00000040:01000000:2.0:1789760745.178027:0:5435:0:(llog_obd.c:197:llog_setup()) obd lustre-MDT0000-mdc-ffffa06c18a1e000 ctxt 13 is initialized 00000020:00000080:1.0:1789760745.180423:0:5435:0:(obd_config.c:826:class_setup()) finished setup of obd lustre-MDT0000-mdc-ffffa06c18a1e000 (uuid 734aaee5-e21a-40f9-9d4e-90e3c76738d2) 00000020:01000000:1.0:1789760745.180444:0:5435:0:(obd_config.c:2004:class_config_llog_handler()) cmd cf014, instance name: lustre-clilmv-ffffa06c18a1e000 00000020:00000080:1.0:1789760745.180450:0:5435:0:(obd_config.c:1472:class_process_config()) processing cmd: cf014 00800000:01000000:1.0:1789760745.180478:0:5435:0:(lmv_obd.c:442:lmv_add_target()) Target uuid: lustre-MDT0000_UUID. index 0 00000020:01000000:1.0:1789760745.180523:0:5435:0:(obd_config.c:1893:class_config_llog_handler()) Marker, inst_flg=0x2 mark_flg=0x2 00000020:00000080:1.0:1789760745.180529:0:5435:0:(obd_config.c:1472:class_process_config()) processing cmd: cf010 00000020:00000080:1.0:1789760745.180531:0:5435:0:(obd_config.c:1563:class_process_config()) marker 7 (0x2) lustre-MDT0000 add mdc 00000020:01000000:1.0:1789760745.180533:0:5435:0:(obd_config.c:1893:class_config_llog_handler()) Marker, inst_flg=0x0 mark_flg=0x5 00000020:01000000:1.0:1789760745.181274:0:5435:0:(obd_config.c:1902:class_config_llog_handler()) SKIP #9 00000020:01000000:1.0:1789760745.181278:0:5435:0:(obd_config.c:1928:class_config_llog_handler()) skipping 0x6 00000020:01000000:1.0:1789760745.181283:0:5435:0:(obd_config.c:1928:class_config_llog_handler()) skipping 0x6 00000020:01000000:1.0:1789760745.181286:0:5435:0:(obd_config.c:1893:class_config_llog_handler()) Marker, inst_flg=0x6 mark_flg=0x6 00000020:00000080:1.0:1789760745.181293:0:5435:0:(obd_config.c:1472:class_process_config()) processing cmd: cf010 00000020:00000080:1.0:1789760745.181295:0:5435:0:(obd_config.c:1563:class_process_config()) marker 9 (0x6) lustre sys.timeout 00000020:01000000:1.0:1789760745.181301:0:5435:0:(obd_config.c:1893:class_config_llog_handler()) Marker, inst_flg=0x0 mark_flg=0x1 00000020:01000004:1.0:1789760745.181305:0:5435:0:(obd_mount.c:961:lustre_check_exclusion()) Check exclusion lustre-OST0000 (0) in 0 of 192.168.203.107@tcp:/lustre 00000020:00000080:1.0:1789760745.181310:0:5435:0:(obd_config.c:1472:class_process_config()) processing cmd: cf010 00000020:00000080:1.0:1789760745.181312:0:5435:0:(obd_config.c:1563:class_process_config()) marker 13 (0x1) lustre-OST0000 add osc 00000020:00000080:1.0:1789760745.181318:0:5435:0:(obd_config.c:1472:class_process_config()) processing cmd: cf005 00000020:00000080:1.0:1789760745.181324:0:5435:0:(obd_config.c:1487:class_process_config()) adding mapping from uuid 192.168.203.107@tcp to nid 0x20000c0a8cb6b (192.168.203.107@tcp) 00000020:01000000:1.0:1789760745.181361:0:5435:0:(obd_config.c:2004:class_config_llog_handler()) cmd cf001, instance name: lustre-OST0000-osc-ffffa06c18a1e000 00000020:00000080:1.0:1789760745.181366:0:5435:0:(obd_config.c:1472:class_process_config()) processing cmd: cf001 00000020:00000080:1.0:1789760745.181391:0:5435:0:(genops.c:413:class_newdev()) Allocate new device lustre-OST0000-osc-ffffa06c18a1e000 (00000000fa31dfea) 00000020:00000080:1.0:1789760745.181409:0:5435:0:(obd_config.c:720:class_attach_name()) OBD: dev 4 attached type osc with refcount 2 00000020:01000000:1.0:1789760745.181418:0:5435:0:(obd_config.c:2004:class_config_llog_handler()) cmd cf003, instance name: lustre-OST0000-osc-ffffa06c18a1e000 00000020:00000080:1.0:1789760745.181422:0:5435:0:(obd_config.c:1472:class_process_config()) processing cmd: cf003 00010000:00080000:1.0:1789760745.181536:0:5435:0:(ldlm_lib.c:102:import_set_conn()) imp 000000007e7480ba@lustre-OST0000-osc-ffffa06c18a1e000: add connection 192.168.203.107@tcp at head 00010000:00010000:1.0:1789760745.182299:0:5435:0:(ldlm_pool.c:923:ldlm_pool_init()) Lock pool ldlm-pool-lustre-OST0000-osc-ffffa06c18a1e000-2 is initialized 00000020:00000080:1.0:1789760745.192985:0:5435:0:(obd_config.c:826:class_setup()) finished setup of obd lustre-OST0000-osc-ffffa06c18a1e000 (uuid 734aaee5-e21a-40f9-9d4e-90e3c76738d2) 00000020:01000000:1.0:1789760745.193011:0:5435:0:(obd_config.c:2004:class_config_llog_handler()) cmd cf00d, instance name: lustre-clilov-ffffa06c18a1e000 00000020:00000080:1.0:1789760745.193026:0:5435:0:(obd_config.c:1472:class_process_config()) processing cmd: cf00d 00020000:01000000:1.0:1789760745.193050:0:5435:0:(lov_obd.c:481:lov_add_target()) uuid:lustre-OST0000_UUID idx:0 gen:1 active:1 00020000:01000000:1.0:1789760745.193059:0:5435:0:(lov_obd.c:510:lov_add_target()) lustre-clilov-ffffa06c18a1e000: Adding target lustre-OST0000_UUID 00020000:01000000:1.0:1789760745.193067:0:5435:0:(lov_obd.c:532:lov_add_target()) idx=0 ltd_gen=1 ld_tgt_count=1 00000020:01000000:1.0:1789760745.193080:0:5435:0:(obd_config.c:1893:class_config_llog_handler()) Marker, inst_flg=0x2 mark_flg=0x2 00000020:00000080:1.0:1789760745.193084:0:5435:0:(obd_config.c:1472:class_process_config()) processing cmd: cf010 00000020:00000080:1.0:1789760745.193086:0:5435:0:(obd_config.c:1563:class_process_config()) marker 13 (0x2) lustre-OST0000 add osc 00000020:01000000:1.0:1789760745.193089:0:5435:0:(obd_config.c:1893:class_config_llog_handler()) Marker, inst_flg=0x0 mark_flg=0x5 00000020:01000000:1.0:1789760745.193091:0:5435:0:(obd_config.c:1902:class_config_llog_handler()) SKIP #16 00000020:01000000:1.0:1789760745.193093:0:5435:0:(obd_config.c:1928:class_config_llog_handler()) skipping 0x6 00000020:01000000:1.0:1789760745.193104:0:5435:0:(obd_config.c:1928:class_config_llog_handler()) skipping 0x6 00000020:01000000:1.0:1789760745.193105:0:5435:0:(obd_config.c:1893:class_config_llog_handler()) Marker, inst_flg=0x6 mark_flg=0x6 00000020:00000080:1.0:1789760745.193108:0:5435:0:(obd_config.c:1472:class_process_config()) processing cmd: cf010 00000020:00000080:1.0:1789760745.193110:0:5435:0:(obd_config.c:1563:class_process_config()) marker 16 (0x6) lustre sys.timeout 00000020:01000000:1.0:1789760745.193112:0:5435:0:(obd_config.c:1893:class_config_llog_handler()) Marker, inst_flg=0x0 mark_flg=0x1 00000020:01000004:1.0:1789760745.193117:0:5435:0:(obd_mount.c:961:lustre_check_exclusion()) Check exclusion lustre-OST0001 (1) in 0 of 192.168.203.107@tcp:/lustre 00000020:00000080:1.0:1789760745.193120:0:5435:0:(obd_config.c:1472:class_process_config()) processing cmd: cf010 00000020:00000080:1.0:1789760745.193121:0:5435:0:(obd_config.c:1563:class_process_config()) marker 19 (0x1) lustre-OST0001 add osc 00000020:00000080:1.0:1789760745.193124:0:5435:0:(obd_config.c:1472:class_process_config()) processing cmd: cf005 00000020:00000080:1.0:1789760745.193129:0:5435:0:(obd_config.c:1487:class_process_config()) adding mapping from uuid 192.168.203.107@tcp to nid 0x20000c0a8cb6b (192.168.203.107@tcp) 00000020:01000000:1.0:1789760745.193157:0:5435:0:(obd_config.c:2004:class_config_llog_handler()) cmd cf001, instance name: lustre-OST0001-osc-ffffa06c18a1e000 00000020:00000080:1.0:1789760745.193160:0:5435:0:(obd_config.c:1472:class_process_config()) processing cmd: cf001 00000020:00000080:1.0:1789760745.193169:0:5435:0:(genops.c:413:class_newdev()) Allocate new device lustre-OST0001-osc-ffffa06c18a1e000 (000000008d2287c8) 00000020:00000080:1.0:1789760745.193183:0:5435:0:(obd_config.c:720:class_attach_name()) OBD: dev 5 attached type osc with refcount 2 00000020:01000000:1.0:1789760745.193186:0:5435:0:(obd_config.c:2004:class_config_llog_handler()) cmd cf003, instance name: lustre-OST0001-osc-ffffa06c18a1e000 00000020:00000080:1.0:1789760745.193188:0:5435:0:(obd_config.c:1472:class_process_config()) processing cmd: cf003 00010000:00080000:1.0:1789760745.193269:0:5435:0:(ldlm_lib.c:102:import_set_conn()) imp 00000000e41e2842@lustre-OST0001-osc-ffffa06c18a1e000: add connection 192.168.203.107@tcp at head 00010000:00010000:1.0:1789760745.193529:0:5435:0:(ldlm_pool.c:923:ldlm_pool_init()) Lock pool ldlm-pool-lustre-OST0001-osc-ffffa06c18a1e000-3 is initialized 00000020:00000080:1.0:1789760745.195142:0:5435:0:(obd_config.c:826:class_setup()) finished setup of obd lustre-OST0001-osc-ffffa06c18a1e000 (uuid 734aaee5-e21a-40f9-9d4e-90e3c76738d2) 00000020:01000000:1.0:1789760745.195157:0:5435:0:(obd_config.c:2004:class_config_llog_handler()) cmd cf00d, instance name: lustre-clilov-ffffa06c18a1e000 00000020:00000080:1.0:1789760745.195161:0:5435:0:(obd_config.c:1472:class_process_config()) processing cmd: cf00d 00020000:01000000:1.0:1789760745.195180:0:5435:0:(lov_obd.c:481:lov_add_target()) uuid:lustre-OST0001_UUID idx:1 gen:1 active:1 00020000:01000000:1.0:1789760745.195194:0:5435:0:(lov_obd.c:510:lov_add_target()) lustre-clilov-ffffa06c18a1e000: Adding target lustre-OST0001_UUID 00020000:01000000:1.0:1789760745.195199:0:5435:0:(lov_obd.c:532:lov_add_target()) idx=1 ltd_gen=1 ld_tgt_count=2 00000020:01000000:1.0:1789760745.195211:0:5435:0:(obd_config.c:1893:class_config_llog_handler()) Marker, inst_flg=0x2 mark_flg=0x2 00000020:00000080:1.0:1789760745.195216:0:5435:0:(obd_config.c:1472:class_process_config()) processing cmd: cf010 00000020:00000080:1.0:1789760745.195217:0:5435:0:(obd_config.c:1563:class_process_config()) marker 19 (0x2) lustre-OST0001 add osc 00000020:01000000:1.0:1789760745.195220:0:5435:0:(obd_config.c:1893:class_config_llog_handler()) Marker, inst_flg=0x0 mark_flg=0x1 00000020:00000080:1.0:1789760745.195223:0:5435:0:(obd_config.c:1472:class_process_config()) processing cmd: cf010 00000020:00000080:1.0:1789760745.195224:0:5435:0:(obd_config.c:1563:class_process_config()) marker 22 (0x1) lustre sys.timeout 00000020:00000080:1.0:1789760745.195227:0:5435:0:(obd_config.c:1472:class_process_config()) processing cmd: cf009 00000020:00000080:1.0:1789760745.195228:0:5435:0:(obd_config.c:1537:class_process_config()) changing lustre timeout from 100 to 20 00000100:00100000:1.0:1789760745.195273:0:5435:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@0000000069d6e114 pname:cluuid:pid:xid:nid:opc:job llog_process_th:02bc4b43-5e6f-4d7e-9224-80f5c789104f:5435:1876700024538112:192.168.203.107@tcp:502: 00000100:00100000:1.0:1789760745.195385:0:5435:0:(client.c:2715:ptlrpc_set_wait()) set 00000000554f0793 going to sleep for 11 seconds 00000100:00100000:2.0:1789760745.201327:0:5435:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@0000000069d6e114 pname:cluuid:pid:xid:nid:opc:job llog_process_th:02bc4b43-5e6f-4d7e-9224-80f5c789104f:5435:1876700024538112:192.168.203.107@tcp:502: 00000020:01000000:2.0:1789760745.201391:0:5435:0:(obd_config.c:1893:class_config_llog_handler()) Marker, inst_flg=0x2 mark_flg=0x2 00000020:00000080:2.0:1789760745.201401:0:5435:0:(obd_config.c:1472:class_process_config()) processing cmd: cf010 00000020:00000080:2.0:1789760745.201404:0:5435:0:(obd_config.c:1563:class_process_config()) marker 22 (0x2) lustre sys.timeout 00000040:00080000:2.0:1789760745.201408:0:5435:0:(llog.c:862:llog_process_thread()) stop processing plain [0xa:0x3:0x0] index 39 count 39 00000020:01000000:3.0:1789760745.201515:0:5415:0:(obd_config.c:2145:class_config_parse_llog()) Processed log lustre-client gen 1-38 (rc=0) 10000000:01000000:3.0:1789760745.201557:0:5415:0:(mgc_request.c:1879:mgc_process_log()) MGC192.168.203.107@tcp: configuration from log 'lustre-client' succeeded (0). 00010000:00010000:3.0:1789760745.201648:0:5415:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.203.107@tcp lock: 000000003a95d4a8/0x59bc99bb6194805f lrc: 3/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x11000000000000 nid: local remote: 0x1daf3002fe9e0867 expref: -99 pid: 5415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1789760745.201661:0:5415:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: MGC192.168.203.107@tcp lock: 000000003a95d4a8/0x59bc99bb6194805f lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x11000000000000 nid: local remote: 0x1daf3002fe9e0867 expref: -99 pid: 5415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:3.0:1789760745.201669:0:5415:0:(mgc_request.c:1802:mgc_process_log()) Process log lustre-cliir-ffffa06c18a1e000 from 1 10000000:01000000:3.0:1789760745.201672:0:5415:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-cliir (res 0x65727473756c) 00010000:00010000:3.0:1789760745.201716:0:5415:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.203.107@tcp lock: 000000005ffd1630/0x59bc99bb61948089 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 5415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1789760745.201724:0:5415:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.203.107@tcp lock: 000000005ffd1630/0x59bc99bb61948089 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 5415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1789760745.201754:0:5415:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.203.107@tcp lock: 000000005ffd1630/0x59bc99bb61948089 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 5415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1789760745.201780:0:5415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@000000002517a04c pname:cluuid:pid:xid:nid:opc:job mount.lustre:02bc4b43-5e6f-4d7e-9224-80f5c789104f:5415:1876700024538240:192.168.203.107@tcp:101: 00000100:00100000:3.0:1789760745.201936:0:5415:0:(client.c:2715:ptlrpc_set_wait()) set 00000000b6784fb3 going to sleep for 11 seconds 00000100:00100000:3.0:1789760745.210008:0:5415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@000000002517a04c pname:cluuid:pid:xid:nid:opc:job mount.lustre:02bc4b43-5e6f-4d7e-9224-80f5c789104f:5415:1876700024538240:192.168.203.107@tcp:101: 00010000:00010000:3.0:1789760745.210040:0:5415:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: MGC192.168.203.107@tcp lock: 000000005ffd1630/0x59bc99bb61948089 lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x1daf3002fe9e086e expref: -99 pid: 5415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1789760745.210053:0:5415:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: MGC192.168.203.107@tcp lock: 000000005ffd1630/0x59bc99bb61948089 lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x1daf3002fe9e086e expref: -99 pid: 5415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1789760745.210647:0:5415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@000000002517a04c pname:cluuid:pid:xid:nid:opc:job mount.lustre:02bc4b43-5e6f-4d7e-9224-80f5c789104f:5415:1876700024538368:192.168.203.107@tcp:256: 00000100:00100000:3.0:1789760745.210749:0:5415:0:(client.c:2715:ptlrpc_set_wait()) set 00000000b6784fb3 going to sleep for 11 seconds 00000100:00100000:3.0:1789760745.221573:0:5415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@000000002517a04c pname:cluuid:pid:xid:nid:opc:job mount.lustre:02bc4b43-5e6f-4d7e-9224-80f5c789104f:5415:1876700024538368:192.168.203.107@tcp:256: 10000000:01000000:3.0:1789760745.222423:0:5415:0:(mgc_request.c:1879:mgc_process_log()) MGC192.168.203.107@tcp: configuration from log 'lustre-cliir' succeeded (0). 00010000:00010000:3.0:1789760745.222454:0:5415:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.203.107@tcp lock: 000000005ffd1630/0x59bc99bb61948089 lrc: 3/1,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x1daf3002fe9e086e expref: -99 pid: 5415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1789760745.222468:0:5415:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: MGC192.168.203.107@tcp lock: 000000005ffd1630/0x59bc99bb61948089 lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x1daf3002fe9e086e expref: -99 pid: 5415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:3.0:1789760745.222477:0:5415:0:(mgc_request.c:1802:mgc_process_log()) Process log params-ffffa06c18a1e000 from 1 10000000:01000000:3.0:1789760745.222480:0:5415:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for params (res 0x736d61726170) 00010000:00010000:3.0:1789760745.222526:0:5415:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.203.107@tcp lock: 00000000155bf639/0x59bc99bb61948090 lrc: 3/1,0 mode: --/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 5415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1789760745.222548:0:5415:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.203.107@tcp lock: 00000000155bf639/0x59bc99bb61948090 lrc: 3/1,0 mode: --/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 5415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1789760745.222559:0:5415:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.203.107@tcp lock: 00000000155bf639/0x59bc99bb61948090 lrc: 3/1,0 mode: --/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 5415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1789760745.222581:0:5415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@000000002517a04c pname:cluuid:pid:xid:nid:opc:job mount.lustre:02bc4b43-5e6f-4d7e-9224-80f5c789104f:5415:1876700024538496:192.168.203.107@tcp:101: 00000100:00100000:3.0:1789760745.222682:0:5415:0:(client.c:2715:ptlrpc_set_wait()) set 00000000b6784fb3 going to sleep for 11 seconds 00000100:00100000:3.0:1789760745.230720:0:5415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@000000002517a04c pname:cluuid:pid:xid:nid:opc:job mount.lustre:02bc4b43-5e6f-4d7e-9224-80f5c789104f:5415:1876700024538496:192.168.203.107@tcp:101: 00010000:00010000:3.0:1789760745.230760:0:5415:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: MGC192.168.203.107@tcp lock: 00000000155bf639/0x59bc99bb61948090 lrc: 4/1,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x1daf3002fe9e0875 expref: -99 pid: 5415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1789760745.230776:0:5415:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: MGC192.168.203.107@tcp lock: 00000000155bf639/0x59bc99bb61948090 lrc: 4/1,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x1daf3002fe9e0875 expref: -99 pid: 5415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1789760745.230894:0:5415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@000000002517a04c pname:cluuid:pid:xid:nid:opc:job mount.lustre:02bc4b43-5e6f-4d7e-9224-80f5c789104f:5415:1876700024538624:192.168.203.107@tcp:501: 00000100:00100000:3.0:1789760745.231024:0:5415:0:(client.c:2715:ptlrpc_set_wait()) set 00000000b6784fb3 going to sleep for 11 seconds 00000100:00100000:3.0:1789760745.242348:0:5415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@000000002517a04c pname:cluuid:pid:xid:nid:opc:job mount.lustre:02bc4b43-5e6f-4d7e-9224-80f5c789104f:5415:1876700024538624:192.168.203.107@tcp:501: 00000100:00100000:3.0:1789760745.242444:0:5415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@000000002517a04c pname:cluuid:pid:xid:nid:opc:job mount.lustre:02bc4b43-5e6f-4d7e-9224-80f5c789104f:5415:1876700024538752:192.168.203.107@tcp:503: 00000100:00100000:3.0:1789760745.242597:0:5415:0:(client.c:2715:ptlrpc_set_wait()) set 00000000b6784fb3 going to sleep for 11 seconds 00000100:00100000:3.0:1789760745.247348:0:5415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@000000002517a04c pname:cluuid:pid:xid:nid:opc:job mount.lustre:02bc4b43-5e6f-4d7e-9224-80f5c789104f:5415:1876700024538752:192.168.203.107@tcp:503: 00000100:00100000:2.0:1789760745.248071:0:5439:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@00000000fd4aa6be pname:cluuid:pid:xid:nid:opc:job llog_process_th:02bc4b43-5e6f-4d7e-9224-80f5c789104f:5439:1876700024538880:192.168.203.107@tcp:502: 00000100:00100000:2.0:1789760745.248209:0:5439:0:(client.c:2715:ptlrpc_set_wait()) set 00000000471e2cd5 going to sleep for 11 seconds 00000100:00100000:2.0:1789760745.253908:0:5439:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@00000000fd4aa6be pname:cluuid:pid:xid:nid:opc:job llog_process_th:02bc4b43-5e6f-4d7e-9224-80f5c789104f:5439:1876700024538880:192.168.203.107@tcp:502: 00000020:01000000:2.0:1789760745.253984:0:5439:0:(obd_config.c:1893:class_config_llog_handler()) Marker, inst_flg=0x0 mark_flg=0x1 00000020:00000080:2.0:1789760745.254001:0:5439:0:(obd_config.c:1472:class_process_config()) processing cmd: cf010 00000020:00000080:2.0:1789760745.254137:0:5439:0:(obd_config.c:1563:class_process_config()) marker 2 (0x1) general debug_raw_pointers 00000020:01000000:2.0:1789760745.254147:0:5439:0:(obd_config.c:2004:class_config_llog_handler()) cmd ce032, instance name: general-ffffa06c18a1e000 00000020:00000080:2.0:1789760745.254150:0:5439:0:(obd_config.c:1472:class_process_config()) processing cmd: ce032 00000100:00100000:2.0:1789760745.254229:0:5439:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51500 pname:cluuid:pid:xid:nid:opc:job llog_process_th:02bc4b43-5e6f-4d7e-9224-80f5c789104f:5439:1876700024539008:192.168.203.107@tcp:502: 00000100:00100000:2.0:1789760745.254414:0:5439:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c091f9cc0 going to sleep for 11 seconds 00000100:00100000:2.0:1789760745.268090:0:5439:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51500 pname:cluuid:pid:xid:nid:opc:job llog_process_th:02bc4b43-5e6f-4d7e-9224-80f5c789104f:5439:1876700024539008:192.168.203.107@tcp:502: 00000020:01000000:2.0:1789760745.268172:0:5439:0:(obd_config.c:1893:class_config_llog_handler()) Marker, inst_flg=0x2 mark_flg=0x2 00000020:00000080:2.0:1789760745.268188:0:5439:0:(obd_config.c:1472:class_process_config()) processing cmd: cf010 00000020:00000080:2.0:1789760745.268191:0:5439:0:(obd_config.c:1563:class_process_config()) marker 2 (0x2) general debug_raw_pointers 00000040:00080000:2.0:1789760745.268209:0:5439:0:(llog.c:862:llog_process_thread()) stop processing plain [0xa:0x2:0x0] index 4 count 4 00000020:01000000:3.0:1789760745.268766:0:5415:0:(obd_config.c:2145:class_config_parse_llog()) Processed log params gen 1-3 (rc=0) 10000000:01000000:3.0:1789760745.268787:0:5415:0:(mgc_request.c:1879:mgc_process_log()) MGC192.168.203.107@tcp: configuration from log 'params' succeeded (0). 00010000:00010000:3.0:1789760745.268882:0:5415:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.203.107@tcp lock: ffffa06c42175600/0x59bc99bb61948090 lrc: 3/1,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x1daf3002fe9e0875 expref: -99 pid: 5415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1789760745.268905:0:5415:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: MGC192.168.203.107@tcp lock: ffffa06c42175600/0x59bc99bb61948090 lrc: 2/0,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x1daf3002fe9e0875 expref: -99 pid: 5415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000080:01000000:3.0:1789760745.268933:0:5415:0:(llite_lib.c:1614:ll_fill_super()) Found profile lustre-client: mdc=lustre-clilmv osc=lustre-clilov 00000020:00000080:3.0:1789760745.268975:0:5415:0:(genops.c:1385:class_connect()) connect: client 734aaee5-e21a-40f9-9d4e-90e3c76738d2, cookie 0x59bc99bb61948097 00800000:01000000:3.0:1789760745.269006:0:5415:0:(lmv_obd.c:515:lmv_check_connect()) Time to connect 734aaee5-e21a-40f9-9d4e-90e3c76738d2 to lustre-clilmv-ffffa06c18a1e000 00800000:01000000:3.0:1789760745.269014:0:5415:0:(lmv_obd.c:338:lmv_connect_mdc()) connect to lustre-MDT0000-mdc-ffffa06c18a1e000(734aaee5-e21a-40f9-9d4e-90e3c76738d2) - lustre-MDT0000_UUID, 734aaee5-e21a-40f9-9d4e-90e3c76738d2 00000020:00000080:3.0:1789760745.269023:0:5415:0:(genops.c:1385:class_connect()) connect: client 734aaee5-e21a-40f9-9d4e-90e3c76738d2, cookie 0x59bc99bb6194809e 00000100:00080000:3.0:1789760745.269034:0:5415:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 0 00000100:00080000:3.0:1789760745.269046:0:5415:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789760745.269061:0:5415:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 134217728, lost: 0. 00000100:00100000:3.0:1789760745.269092:0:5415:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872fb80 x1876700024539136/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789760745.269133:0:5415:0:(pinger.c:373:ptlrpc_pinger_add_import()) adding pingable import 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID 00800000:01000000:3.0:1789760745.269247:0:5415:0:(lmv_obd.c:392:lmv_connect_mdc()) Connected to lustre-MDT0000-mdc-ffffa06c18a1e000(734aaee5-e21a-40f9-9d4e-90e3c76738d2) successfully (5) 00800000:00000004:3.0:1789760745.269251:0:5415:0:(obd_class.h:1056:obd_statfs_async()) lustre-MDT0000-mdc-ffffa06c18a1e000: age -816, max_age 0 00000080:00000004:3.0:1789760745.269286:0:5415:0:(obd_class.h:1090:obd_statfs()) lustre-clilmv-ffffa06c18a1e000: age -816, max_age 183 00800000:00000004:3.0:1789760745.269293:0:5415:0:(obd_class.h:1090:obd_statfs()) lustre-MDT0000-mdc-ffffa06c18a1e000: age -816, max_age 183 00000100:00080000:3.0:1789760745.269312:0:5415:0:(client.c:1845:ptlrpc_send_new_req()) @@@ req waiting for recovery: (FULL != CONNECTING) req@ffffa06c1872f480 x1876700024539392/t0(0) o41->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 224/368 e 0 to 0 dl 0 ref 2 fl Rpc:WQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789760745.269323:0:5415:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c08e7b240 going to sleep for 0 seconds 00000100:00100000:1.0:1789760745.269905:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024539136:192.168.203.107@tcp:38: 00000100:00080000:2.0:1789760745.270707:0:2344:0:(client.c:1845:ptlrpc_send_new_req()) @@@ req waiting for recovery: (FULL != CONNECTING) req@ffffa06c1872e680 x1876700024539264/t0(0) o41->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 224/368 e 0 to 0 dl 0 ref 1 fl Rpc:WQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00010000:00080000:3.0:1789760745.292731:0:2340:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61549) req@ffffa06c1872fb80 x1876700024539136/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/416 e 0 to 0 dl 1789760755 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789760745.292748:0:2340:0:(client.c:3125:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffffa06c18a1e000: committing for last_committed 0 gen 1 00000100:00080000:3.0:1789760745.292773:0:2340:0:(import.c:1132:ptlrpc_connect_interpret()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to target with instance 1 00000001:01000000:3.0:1789760745.542026:0:2340:0:(integrity.c:236:obd_t10_performance_test()) lustre-MDT0000-mdc-ffffa06c18a1e000: T10 checksum algorithm t10ip512 speed = 2559 MB/s 00000001:01000000:3.0:1789760745.792241:0:2340:0:(integrity.c:236:obd_t10_performance_test()) lustre-MDT0000-mdc-ffffa06c18a1e000: T10 checksum algorithm t10ip4K speed = 5895 MB/s 00000001:01000000:2.0:1789760746.049361:0:2340:0:(integrity.c:236:obd_t10_performance_test()) lustre-MDT0000-mdc-ffffa06c18a1e000: T10 checksum algorithm t10crc512 speed = 719 MB/s 00010000:00010000:1.0:1789760746.103281:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789760746.295646:0:5415:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c08e7b240 going to sleep for 0 seconds 00000001:01000000:3.0:1789760746.299099:0:2340:0:(integrity.c:236:obd_t10_performance_test()) lustre-MDT0000-mdc-ffffa06c18a1e000: T10 checksum algorithm t10crc4K speed = 767 MB/s 00000100:00080000:3.0:1789760746.299113:0:2340:0:(import.c:956:ptlrpc_connect_set_flags()) lustre-MDT0000-mdc-ffffa06c18a1e000: Resetting ns_connect_flags to server flags: 0xae7a7e7be344d3b8 00000100:00080000:3.0:1789760746.299128:0:2340:0:(import.c:1213:ptlrpc_connect_interpret()) connected to replayable target: lustre-MDT0000_UUID 00000100:00080000:3.0:1789760746.299134:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to FULL 00000100:00080000:1.0:1789760746.300764:0:2340:0:(recover.c:191:ptlrpc_wake_delayed()) @@@ waking (set 00000000fc5b1d38): req@ffffa06c1872e680 x1876700024539264/t0(0) o41->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 224/368 e 0 to 0 dl 0 ref 1 fl Rpc:WQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789760746.300791:0:2340:0:(recover.c:191:ptlrpc_wake_delayed()) @@@ waking (set 00000000046e5f60): req@ffffa06c1872f480 x1876700024539392/t0(0) o41->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 224/368 e 0 to 0 dl 0 ref 2 fl Rpc:WQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789760746.300815:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024539136:192.168.203.107@tcp:38: 00000100:00100000:0.0:1789760746.302934:0:5415:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c08e7b240 going to sleep for 11 seconds 00010000:00080000:2.0:1789760746.305193:0:2344:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61549) req@ffffa06c1872e680 x1876700024539264/t0(0) o41->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 224/368 e 0 to 0 dl 1789760762 ref 1 fl Rpc:RQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789760746.305218:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:734aaee5-e21a-40f9-9d4e-90e3c76738d2:0:1876700024539264:192.168.203.107@tcp:41: 00010000:00080000:0.0:1789760746.313637:0:5415:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61549) req@ffffa06c1872f480 x1876700024539392/t0(0) o41->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 224/368 e 0 to 0 dl 1789760762 ref 2 fl Rpc:RQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760746.313658:0:5415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872f480 pname:cluuid:pid:xid:nid:opc:job mount.lustre:734aaee5-e21a-40f9-9d4e-90e3c76738d2:5415:1876700024539392:192.168.203.107@tcp:41: 00800000:00000004:0.0:1789760746.313683:0:5415:0:(obd_class.h:1116:obd_statfs()) lustre-MDT0000-mdc-ffffa06c18a1e000: update ffffa06c04e5b7c0 cache blocks 317407/353529 objects 1023725/1024000 00000080:00000004:0.0:1789760746.313691:0:5415:0:(obd_class.h:1116:obd_statfs()) lustre-clilmv-ffffa06c18a1e000: update ffffa06c04e58450 cache blocks 317407/353529 objects 1023725/1024000 00000100:00100000:0.0:1789760746.313744:0:5415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0000 pname:cluuid:pid:xid:nid:opc:job mount.lustre:734aaee5-e21a-40f9-9d4e-90e3c76738d2:5415:1876700024539520:192.168.203.107@tcp:40: 00000100:00100000:0.0:1789760746.313835:0:5415:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c1870c300 going to sleep for 11 seconds 00010000:00080000:0.0:1789760746.316804:0:5415:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61549) req@ffffa06c091d0000 x1876700024539520/t0(0) o40->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 440/448 e 0 to 0 dl 1789760762 ref 2 fl Rpc:RQU/209/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760746.316826:0:5415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0000 pname:cluuid:pid:xid:nid:opc:job mount.lustre:734aaee5-e21a-40f9-9d4e-90e3c76738d2:5415:1876700024539520:192.168.203.107@tcp:40: 00000080:00000004:0.0:1789760746.316901:0:5415:0:(llite_lib.c:614:client_common_fill_super()) rootfid [0x200000007:0x1:0x0] 00000080:00100000:0.0:1789760746.316916:0:5415:0:(llite_lib.c:775:client_common_fill_super()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 2146304 00020000:01000000:0.0:1789760746.316920:0:5415:0:(lov_obd.c:186:lov_connect()) connect #0 00000020:00000080:0.0:1789760746.316932:0:5415:0:(genops.c:1385:class_connect()) connect: client 734aaee5-e21a-40f9-9d4e-90e3c76738d2, cookie 0x59bc99bb619480a5 00000020:00000080:0.0:1789760746.316950:0:5415:0:(genops.c:1385:class_connect()) connect: client LOV_OSC_UUID, cookie 0x59bc99bb619480ac 00000100:00080000:0.0:1789760746.316958:0:5415:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 0 00000100:00080000:0.0:1789760746.316965:0:5415:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffffa06c18a1e000: import ffffa06c107c9800 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789760746.316977:0:5415:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 134217728, lost: 0. 00000100:00100000:0.0:1789760746.317002:0:5415:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0000 x1876700024539648/t0(0) o8->lustre-OST0000-osc-ffffa06c18a1e000@192.168.203.107@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789760746.317019:0:5415:0:(pinger.c:373:ptlrpc_pinger_add_import()) adding pingable import 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-OST0000_UUID 00020000:01000000:0.0:1789760746.317042:0:5415:0:(lov_obd.c:159:lov_connect_osc()) Connected tgt idx 0 lustre-OST0000_UUID (lustre-OST0000-osc-ffffa06c18a1e000) active 00000020:00000080:0.0:1789760746.317069:0:5415:0:(genops.c:1385:class_connect()) connect: client LOV_OSC_UUID, cookie 0x59bc99bb619480b3 00000100:00080000:0.0:1789760746.317157:0:5415:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 0 00000100:00100000:2.0:1789760746.317157:0:2344:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00080000:0.0:1789760746.317166:0:5415:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffffa06c18a1e000: import ffffa06c0918f000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789760746.317174:0:5415:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 134217728, lost: 0. 00000100:00100000:2.0:1789760746.317179:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2344:1876700024539776:192.168.203.107@tcp:400: 00000100:00100000:0.0:1789760746.317191:0:5415:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d1500 x1876700024539904/t0(0) o8->lustre-OST0001-osc-ffffa06c18a1e000@192.168.203.107@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789760746.317202:0:5415:0:(pinger.c:373:ptlrpc_pinger_add_import()) adding pingable import 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-OST0001_UUID 00020000:01000000:0.0:1789760746.317213:0:5415:0:(lov_obd.c:159:lov_connect_osc()) Connected tgt idx 1 lustre-OST0001_UUID (lustre-OST0001-osc-ffffa06c18a1e000) active 00000100:00100000:0.0:1789760746.317252:0:5415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0e00 pname:cluuid:pid:xid:nid:opc:job mount.lustre:734aaee5-e21a-40f9-9d4e-90e3c76738d2:5415:1876700024540032:192.168.203.107@tcp:33: 00000100:00100000:0.0:1789760746.317337:0:5415:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c1870c300 going to sleep for 11 seconds 00000100:00100000:1.0:1789760746.317640:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024539648:192.168.203.107@tcp:8: 00000100:00100000:1.0:1789760746.317706:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024539904:192.168.203.107@tcp:8: 00000100:00100000:2.0:1789760746.325346:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2344:1876700024539776:192.168.203.107@tcp:400: 00010000:00080000:1.0:1789760746.331984:0:2340:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61549) req@ffffa06c091d0000 x1876700024539648/t0(0) o8->lustre-OST0000-osc-ffffa06c18a1e000@192.168.203.107@tcp:28/4 lens 520/416 e 0 to 0 dl 1789760756 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789760746.332001:0:2340:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffffa06c18a1e000: committing for last_committed 0 gen 1 00000100:00080000:1.0:1789760746.332013:0:2340:0:(import.c:1132:ptlrpc_connect_interpret()) lustre-OST0000-osc-ffffa06c18a1e000: connect to target with instance 1 00000100:00080000:1.0:1789760746.332023:0:2340:0:(import.c:956:ptlrpc_connect_set_flags()) lustre-OST0000-osc-ffffa06c18a1e000: Resetting ns_connect_flags to server flags: 0xa0427af2e3440038 00000100:00080000:1.0:1789760746.332047:0:2340:0:(import.c:1213:ptlrpc_connect_interpret()) connected to replayable target: lustre-OST0000_UUID 00000100:00080000:1.0:1789760746.332050:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c107c9800 lustre-OST0000_UUID: changing import state from CONNECTING to FULL 00020000:01000000:1.0:1789760746.332057:0:2340:0:(lov_obd.c:398:lov_set_osc_active()) Marking OSC lustre-OST0000_UUID active 00000080:00000004:1.0:1789760746.332061:0:2340:0:(lcommon_misc.c:94:cl_ocd_update()) Changing connect_flags: 0xa0467af2e3640038 -> 0xa0427af2e3440038 00000080:00080000:1.0:1789760746.332066:0:2340:0:(lcommon_misc.c:56:cl_init_ea_size()) updating def/max_easize: 72/65536 00000100:00100000:1.0:1789760746.332098:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024539648:192.168.203.107@tcp:8: 00010000:00080000:3.0:1789760746.332408:0:2340:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61549) req@ffffa06c091d1500 x1876700024539904/t0(0) o8->lustre-OST0001-osc-ffffa06c18a1e000@192.168.203.107@tcp:28/4 lens 520/416 e 0 to 0 dl 1789760756 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789760746.332418:0:2340:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0001-osc-ffffa06c18a1e000: committing for last_committed 0 gen 1 00000100:00080000:3.0:1789760746.332422:0:2340:0:(import.c:1132:ptlrpc_connect_interpret()) lustre-OST0001-osc-ffffa06c18a1e000: connect to target with instance 1 00000100:00080000:3.0:1789760746.332427:0:2340:0:(import.c:956:ptlrpc_connect_set_flags()) lustre-OST0001-osc-ffffa06c18a1e000: Resetting ns_connect_flags to server flags: 0xa0427af2e3440038 00000100:00080000:3.0:1789760746.332432:0:2340:0:(import.c:1213:ptlrpc_connect_interpret()) connected to replayable target: lustre-OST0001_UUID 00000100:00080000:3.0:1789760746.332434:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c0918f000 lustre-OST0001_UUID: changing import state from CONNECTING to FULL 00020000:01000000:3.0:1789760746.332438:0:2340:0:(lov_obd.c:398:lov_set_osc_active()) Marking OSC lustre-OST0001_UUID active 00000080:00000004:3.0:1789760746.332440:0:2340:0:(lcommon_misc.c:94:cl_ocd_update()) Changing connect_flags: 0xa0427af2e3440038 -> 0xa0427af2e3440038 00000080:00080000:3.0:1789760746.332443:0:2340:0:(lcommon_misc.c:56:cl_init_ea_size()) updating def/max_easize: 72/65536 00000100:00100000:3.0:1789760746.332449:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024539904:192.168.203.107@tcp:8: 00010000:00080000:1.0:1789760746.371435:0:5415:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61549) req@ffffa06c091d0e00 x1876700024540032/t0(0) o33->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:23/10 lens 448/464 e 0 to 0 dl 1789760762 ref 2 fl Rpc:RQU/600/0 rc 0/0 job:'' uid:0 gid:0 projid:0 00000100:00100000:1.0:1789760746.371456:0:5415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0e00 pname:cluuid:pid:xid:nid:opc:job mount.lustre:734aaee5-e21a-40f9-9d4e-90e3c76738d2:5415:1876700024540032:192.168.203.107@tcp:33: 00000080:00200000:1.0:1789760746.371673:0:5415:0:(llite_lib.c:3356:ll_read_inode2()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908) 00000080:00200000:1.0:1789760746.371682:0:5415:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 4096 00000080:02000400:1.0:1789760746.371752:0:5415:0:(llite_lib.c:1653:ll_fill_super()) Mounted lustre-client - version 2.17.57_103_g082f77f 00000080:00000004:1.0:1789760746.378357:0:5415:0:(super25.c:182:lustre_fill_super()) 192.168.203.107@tcp:/lustre: Mount complete 00000080:00200008:1.0:1789760746.540541:0:5456:0:(file.c:6296:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffffa06c03518908), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1789760746.540554:0:5456:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908),name=/ 00000002:00010000:1.0:1789760746.540577:0:5456:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1789760746.540636:0:5456:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c19d4c000/0x59bc99bb619480ba lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 5456 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1789760746.540647:0:5456:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c19d4c000/0x59bc99bb619480ba lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 5456 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1789760746.540658:0:5456:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c19d4c000/0x59bc99bb619480ba lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 5456 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1789760746.540682:0:5456:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2c700 pname:cluuid:pid:xid:nid:opc:job mkdir:734aaee5-e21a-40f9-9d4e-90e3c76738d2:5456:1876700024540160:192.168.203.107@tcp:101: 00000100:00100000:1.0:1789760746.540950:0:5456:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c04f39300 going to sleep for 11 seconds 00000100:00100000:1.0:1789760746.543525:0:5456:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2c700 pname:cluuid:pid:xid:nid:opc:job mkdir:734aaee5-e21a-40f9-9d4e-90e3c76738d2:5456:1876700024540160:192.168.203.107@tcp:101: 00010000:00010000:1.0:1789760746.543569:0:5456:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c19d4c000/0x59bc99bb619480ba lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1daf3002fe9e0898 expref: -99 pid: 5456 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1789760746.543581:0:5456:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c19d4c000/0x59bc99bb619480ba lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1daf3002fe9e0898 expref: -99 pid: 5456 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1789760746.543591:0:5456:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c19d4c000/0x59bc99bb619480ba lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1daf3002fe9e0898 expref: -99 pid: 5456 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1789760746.543606:0:5456:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c19d4c000/0x59bc99bb619480ba lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1daf3002fe9e0898 expref: -99 pid: 5456 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1789760746.543619:0:5456:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c19d4c000/0x59bc99bb619480ba lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1daf3002fe9e0898 expref: -99 pid: 5456 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1789760746.543628:0:5456:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c19d4c000/0x59bc99bb619480ba lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1daf3002fe9e0898 expref: -99 pid: 5456 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1789760746.543635:0:5456:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c19d4c000/0x59bc99bb619480ba lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1daf3002fe9e0898 expref: -99 pid: 5456 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1789760746.543648:0:5456:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffffa06c08e2c700 x1876700024540160/t0(0) o101->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 576/688 e 0 to 0 dl 1789760762 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'' uid:0 gid:0 projid:0 00000002:00010000:1.0:1789760746.543661:0:5456:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c19d4c000/0x59bc99bb619480ba lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1daf3002fe9e0898 expref: -99 pid: 5456 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1789760746.543674:0:5456:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffffa06c18665000 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:1.0:1789760746.543681:0:5456:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:1.0:1789760746.543713:0:5456:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 4096 00000080:00010000:1.0:1789760746.543718:0:5456:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa06c03518908) for lock 0x59bc99bb619480ba 00000080:00010000:1.0:1789760746.543724:0:5456:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa06c03518908) 00000080:00010000:1.0:1789760746.543727:0:5456:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x59bc99bb619480ba from it ffffafafc3d03cb8 00010000:00010000:1.0:1789760746.543731:0:5456:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c19d4c000/0x59bc99bb619480ba lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1daf3002fe9e0898 expref: -99 pid: 5456 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1789760746.543740:0:5456:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c19d4c000/0x59bc99bb619480ba lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1daf3002fe9e0898 expref: -99 pid: 5456 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1789760747.127871:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789760748.153554:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200008:3.0:1789760748.558166:0:5508:0:(file.c:6296:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffffa06c03518908), request_mask 2047, flags 0, foreign 0 00000080:00200000:3.0:1789760748.558181:0:5508:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908),name=/ 00000002:00010000:3.0:1789760748.558210:0:5508:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1789760748.558230:0:5508:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c19d4c000/0x59bc99bb619480ba lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1daf3002fe9e0898 expref: -99 pid: 5456 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1789760748.558247:0:5508:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c19d4c000/0x59bc99bb619480ba lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1daf3002fe9e0898 expref: -99 pid: 5456 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:3.0:1789760748.558262:0:5508:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1789760748.558270:0:5508:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa06c03518908) 00000080:00010000:3.0:1789760748.558274:0:5508:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa06c03518908) for lock 0x59bc99bb619480ba 00000080:00010000:3.0:1789760748.558281:0:5508:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x59bc99bb619480ba from it ffffafafc3e97cb8 00010000:00010000:3.0:1789760748.558286:0:5508:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c19d4c000/0x59bc99bb619480ba lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1daf3002fe9e0898 expref: -99 pid: 5456 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1789760748.558295:0:5508:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c19d4c000/0x59bc99bb619480ba lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1daf3002fe9e0898 expref: -99 pid: 5456 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 1 00000080:00200008:3.0:1789760748.559252:0:5508:0:(file.c:6296:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffffa06c03518908), request_mask 0, flags 0, foreign 0 00000080:00200000:3.0:1789760748.559259:0:5508:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908),name=/ 00000002:00010000:3.0:1789760748.559270:0:5508:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1789760748.559280:0:5508:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c19d4c000/0x59bc99bb619480ba lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1daf3002fe9e0898 expref: -99 pid: 5456 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 1 00010000:00010000:3.0:1789760748.559291:0:5508:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c19d4c000/0x59bc99bb619480ba lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1daf3002fe9e0898 expref: -99 pid: 5456 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 1 00000080:00200000:3.0:1789760748.559297:0:5508:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1789760748.559302:0:5508:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa06c03518908) 00000080:00010000:3.0:1789760748.559304:0:5508:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa06c03518908) for lock 0x59bc99bb619480ba 00000080:00010000:3.0:1789760748.559310:0:5508:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x59bc99bb619480ba from it ffffafafc3e97cb8 00010000:00010000:3.0:1789760748.559313:0:5508:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c19d4c000/0x59bc99bb619480ba lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1daf3002fe9e0898 expref: -99 pid: 5456 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 1 00010000:00010000:3.0:1789760748.559332:0:5508:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c19d4c000/0x59bc99bb619480ba lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1daf3002fe9e0898 expref: -99 pid: 5456 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 1 00000080:00200000:3.0:1789760748.559457:0:5508:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908),name=/ 00000002:00010000:3.0:1789760748.559465:0:5508:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1789760748.559472:0:5508:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c19d4c000/0x59bc99bb619480ba lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1daf3002fe9e0898 expref: -99 pid: 5456 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 1 00010000:00010000:3.0:1789760748.559481:0:5508:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c19d4c000/0x59bc99bb619480ba lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1daf3002fe9e0898 expref: -99 pid: 5456 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 1 00000080:00200000:3.0:1789760748.559487:0:5508:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1789760748.559491:0:5508:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa06c03518908) 00000080:00010000:3.0:1789760748.559493:0:5508:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa06c03518908) for lock 0x59bc99bb619480ba 00000080:00010000:3.0:1789760748.559497:0:5508:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x59bc99bb619480ba from it ffffafafc3e97bb0 00010000:00010000:3.0:1789760748.559500:0:5508:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c19d4c000/0x59bc99bb619480ba lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1daf3002fe9e0898 expref: -99 pid: 5456 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 1 00010000:00010000:3.0:1789760748.559507:0:5508:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c19d4c000/0x59bc99bb619480ba lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1daf3002fe9e0898 expref: -99 pid: 5456 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 1 00000080:00200000:3.0:1789760748.559515:0:5508:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908), inode mode 41ed mask 44 00000080:00200008:3.0:1789760748.559521:0:5508:0:(file.c:1090:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffffa06c03518908), flags 100000 00000080:00200080:3.0:1789760748.559916:0:5508:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908) cmd=c00866a4 arg=7ffdb86da240 00800000:00000080:3.0:1789760748.559934:0:5508:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06c18a1e000: cmd=c00866a4 len=592 karg=ffffa06c10e89400 uarg=0000000000000000 00800000:00000004:3.0:1789760748.559942:0:5508:0:(obd_class.h:1090:obd_statfs()) lustre-MDT0000-mdc-ffffa06c18a1e000: age 185, max_age 186 00000100:00100000:3.0:1789760748.559987:0:5508:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872ea00 pname:cluuid:pid:xid:nid:opc:job lfs:734aaee5-e21a-40f9-9d4e-90e3c76738d2:5508:1876700024540288:192.168.203.107@tcp:41: 00000100:00100000:3.0:1789760748.560100:0:5508:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c08e7b9c0 going to sleep for 11 seconds 00000100:00100000:3.0:1789760748.566331:0:5508:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872ea00 pname:cluuid:pid:xid:nid:opc:job lfs:734aaee5-e21a-40f9-9d4e-90e3c76738d2:5508:1876700024540288:192.168.203.107@tcp:41: 00800000:00000004:3.0:1789760748.566427:0:5508:0:(obd_class.h:1116:obd_statfs()) lustre-MDT0000-mdc-ffffa06c18a1e000: update ffffa06c04e5b7c0 cache blocks 317407/353529 objects 1023725/1024000 00000080:00200080:3.0:1789760748.566504:0:5508:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908) cmd=c00866a4 arg=7ffdb86da240 00800000:00000080:3.0:1789760748.566520:0:5508:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06c18a1e000: cmd=c00866a4 len=592 karg=ffffa06c197b4400 uarg=0000000000000000 00000080:00200080:3.0:1789760748.566528:0:5508:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908) cmd=c00866a4 arg=7ffdb86da240 00800000:00000080:3.0:1789760748.566532:0:5508:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06c18a1e000: cmd=c00866a4 len=592 karg=ffffa06c197b4400 uarg=0000000000000000 00000080:00200080:3.0:1789760748.566537:0:5508:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908) cmd=c00866a4 arg=7ffdb86da240 00800000:00000080:3.0:1789760748.566541:0:5508:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06c18a1e000: cmd=c00866a4 len=592 karg=ffffa06c197b4400 uarg=0000000000000000 00000080:00200080:3.0:1789760748.566545:0:5508:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908) cmd=c00866a4 arg=7ffdb86da240 00800000:00000080:3.0:1789760748.566548:0:5508:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06c18a1e000: cmd=c00866a4 len=592 karg=ffffa06c197b4400 uarg=0000000000000000 00000080:00200080:3.0:1789760748.566552:0:5508:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908) cmd=c00866a4 arg=7ffdb86da240 00800000:00000080:3.0:1789760748.566555:0:5508:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06c18a1e000: cmd=c00866a4 len=592 karg=ffffa06c197b4400 uarg=0000000000000000 00000080:00200080:3.0:1789760748.566558:0:5508:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908) cmd=c00866a4 arg=7ffdb86da240 00800000:00000080:3.0:1789760748.566561:0:5508:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06c18a1e000: cmd=c00866a4 len=592 karg=ffffa06c197b4400 uarg=0000000000000000 00000080:00200080:3.0:1789760748.566694:0:5508:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908) cmd=c00866a4 arg=7ffdb86da240 00800000:00000080:3.0:1789760748.566700:0:5508:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06c18a1e000: cmd=c00866a4 len=592 karg=ffffa06c197b4400 uarg=0000000000000000 00000080:00200080:3.0:1789760748.566704:0:5508:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908) cmd=c00866a4 arg=7ffdb86da240 00800000:00000080:3.0:1789760748.566707:0:5508:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06c18a1e000: cmd=c00866a4 len=592 karg=ffffa06c197b4400 uarg=0000000000000000 00000080:00200080:3.0:1789760748.566710:0:5508:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908) cmd=c00866a4 arg=7ffdb86da240 00800000:00000080:3.0:1789760748.566713:0:5508:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06c18a1e000: cmd=c00866a4 len=592 karg=ffffa06c197b4400 uarg=0000000000000000 00000080:00200080:3.0:1789760748.566716:0:5508:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908) cmd=c00866a4 arg=7ffdb86da240 00800000:00000080:3.0:1789760748.566719:0:5508:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06c18a1e000: cmd=c00866a4 len=592 karg=ffffa06c197b4400 uarg=0000000000000000 00000080:00200080:3.0:1789760748.566722:0:5508:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908) cmd=c00866a4 arg=7ffdb86da240 00800000:00000080:3.0:1789760748.566742:0:5508:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06c18a1e000: cmd=c00866a4 len=592 karg=ffffa06c197b4400 uarg=0000000000000000 00000080:00200080:3.0:1789760748.566747:0:5508:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908) cmd=c00866a4 arg=7ffdb86da240 00800000:00000080:3.0:1789760748.566750:0:5508:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06c18a1e000: cmd=c00866a4 len=592 karg=ffffa06c197b4400 uarg=0000000000000000 00000080:00200080:3.0:1789760748.566753:0:5508:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908) cmd=c00866a4 arg=7ffdb86da240 00800000:00000080:3.0:1789760748.566756:0:5508:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06c18a1e000: cmd=c00866a4 len=592 karg=ffffa06c197b4400 uarg=0000000000000000 00000080:00200080:3.0:1789760748.566760:0:5508:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908) cmd=c00866a4 arg=7ffdb86da240 00800000:00000080:3.0:1789760748.566764:0:5508:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06c18a1e000: cmd=c00866a4 len=592 karg=ffffa06c197b4400 uarg=0000000000000000 00000080:00200080:3.0:1789760748.566767:0:5508:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908) cmd=c00866a4 arg=7ffdb86da240 00800000:00000080:3.0:1789760748.566770:0:5508:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06c18a1e000: cmd=c00866a4 len=592 karg=ffffa06c197b4400 uarg=0000000000000000 00000080:00200080:3.0:1789760748.566774:0:5508:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908) cmd=c00866a4 arg=7ffdb86da240 00800000:00000080:3.0:1789760748.566777:0:5508:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06c18a1e000: cmd=c00866a4 len=592 karg=ffffa06c197b4400 uarg=0000000000000000 00000080:00200080:3.0:1789760748.566781:0:5508:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908) cmd=c00866a4 arg=7ffdb86da240 00800000:00000080:3.0:1789760748.566784:0:5508:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06c18a1e000: cmd=c00866a4 len=592 karg=ffffa06c197b4400 uarg=0000000000000000 00000080:00200080:3.0:1789760748.566788:0:5508:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908) cmd=c00866a4 arg=7ffdb86da240 00800000:00000080:3.0:1789760748.566792:0:5508:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06c18a1e000: cmd=c00866a4 len=592 karg=ffffa06c197b4400 uarg=0000000000000000 00000080:00200080:3.0:1789760748.566795:0:5508:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908) cmd=c00866a4 arg=7ffdb86da240 00800000:00000080:3.0:1789760748.566799:0:5508:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06c18a1e000: cmd=c00866a4 len=592 karg=ffffa06c197b4400 uarg=0000000000000000 00000080:00200080:3.0:1789760748.566803:0:5508:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908) cmd=c00866a4 arg=7ffdb86da240 00800000:00000080:3.0:1789760748.566806:0:5508:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06c18a1e000: cmd=c00866a4 len=592 karg=ffffa06c197b4400 uarg=0000000000000000 00000080:00200080:3.0:1789760748.566810:0:5508:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908) cmd=c00866a4 arg=7ffdb86da240 00800000:00000080:3.0:1789760748.566813:0:5508:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06c18a1e000: cmd=c00866a4 len=592 karg=ffffa06c197b4400 uarg=0000000000000000 00000080:00200080:3.0:1789760748.566817:0:5508:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908) cmd=c00866a4 arg=7ffdb86da240 00800000:00000080:3.0:1789760748.566820:0:5508:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06c18a1e000: cmd=c00866a4 len=592 karg=ffffa06c197b4400 uarg=0000000000000000 00000080:00200080:3.0:1789760748.566823:0:5508:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908) cmd=c00866a4 arg=7ffdb86da240 00800000:00000080:3.0:1789760748.566827:0:5508:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06c18a1e000: cmd=c00866a4 len=592 karg=ffffa06c197b4400 uarg=0000000000000000 00000080:00200080:3.0:1789760748.566838:0:5508:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908) cmd=c00866a4 arg=7ffdb86da240 00800000:00000080:3.0:1789760748.566843:0:5508:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06c18a1e000: cmd=c00866a4 len=592 karg=ffffa06c197b4400 uarg=0000000000000000 00000080:00200080:3.0:1789760748.566876:0:5508:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908) cmd=c00866a4 arg=7ffdb86da240 00800000:00000080:3.0:1789760748.566881:0:5508:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06c18a1e000: cmd=c00866a4 len=592 karg=ffffa06c197b4400 uarg=0000000000000000 00000080:00200080:3.0:1789760748.566885:0:5508:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908) cmd=c00866a4 arg=7ffdb86da240 00800000:00000080:3.0:1789760748.566888:0:5508:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06c18a1e000: cmd=c00866a4 len=592 karg=ffffa06c197b4400 uarg=0000000000000000 00000080:00200080:3.0:1789760748.566892:0:5508:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908) cmd=c00866a4 arg=7ffdb86da240 00800000:00000080:3.0:1789760748.566895:0:5508:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06c18a1e000: cmd=c00866a4 len=592 karg=ffffa06c197b4400 uarg=0000000000000000 00000080:00200080:3.0:1789760748.566899:0:5508:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908) cmd=c00866a4 arg=7ffdb86da240 00800000:00000080:3.0:1789760748.566902:0:5508:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06c18a1e000: cmd=c00866a4 len=592 karg=ffffa06c197b4400 uarg=0000000000000000 00000080:00200080:3.0:1789760748.566906:0:5508:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908) cmd=c00866a4 arg=7ffdb86da240 00800000:00000080:3.0:1789760748.566910:0:5508:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06c18a1e000: cmd=c00866a4 len=592 karg=ffffa06c197b4400 uarg=0000000000000000 00000080:00200080:3.0:1789760748.566914:0:5508:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908) cmd=c00866a4 arg=7ffdb86da240 00800000:00000080:3.0:1789760748.566917:0:5508:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06c18a1e000: cmd=c00866a4 len=592 karg=ffffa06c197b4400 uarg=0000000000000000 00000080:00200080:3.0:1789760748.566921:0:5508:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908) cmd=c00866a4 arg=7ffdb86da240 00800000:00000080:3.0:1789760748.566924:0:5508:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06c18a1e000: cmd=c00866a4 len=592 karg=ffffa06c197b4400 uarg=0000000000000000 00000080:00200080:3.0:1789760748.566928:0:5508:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908) cmd=c00866a4 arg=7ffdb86da240 00800000:00000080:3.0:1789760748.566932:0:5508:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06c18a1e000: cmd=c00866a4 len=592 karg=ffffa06c197b4400 uarg=0000000000000000 00000080:00200080:3.0:1789760748.566935:0:5508:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908) cmd=c00866a4 arg=7ffdb86da240 00800000:00000080:3.0:1789760748.566939:0:5508:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06c18a1e000: cmd=c00866a4 len=592 karg=ffffa06c197b4400 uarg=0000000000000000 00000080:00200080:3.0:1789760748.566943:0:5508:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908) cmd=c00866a4 arg=7ffdb86da240 00800000:00000080:3.0:1789760748.566946:0:5508:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06c18a1e000: cmd=c00866a4 len=592 karg=ffffa06c197b4400 uarg=0000000000000000 00000080:00200080:3.0:1789760748.566950:0:5508:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908) cmd=c00866a4 arg=7ffdb86da240 00800000:00000080:3.0:1789760748.566953:0:5508:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06c18a1e000: cmd=c00866a4 len=592 karg=ffffa06c197b4400 uarg=0000000000000000 00000080:00200080:3.0:1789760748.566956:0:5508:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908) cmd=c00866a4 arg=7ffdb86da240 00800000:00000080:3.0:1789760748.566968:0:5508:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06c18a1e000: cmd=c00866a4 len=592 karg=ffffa06c197b4400 uarg=0000000000000000 00000080:00200080:3.0:1789760748.566973:0:5508:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908) cmd=c00866a4 arg=7ffdb86da240 00800000:00000080:3.0:1789760748.566976:0:5508:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06c18a1e000: cmd=c00866a4 len=592 karg=ffffa06c197b4400 uarg=0000000000000000 00000080:00200080:3.0:1789760748.566979:0:5508:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908) cmd=c00866a4 arg=7ffdb86da240 00800000:00000080:3.0:1789760748.566982:0:5508:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06c18a1e000: cmd=c00866a4 len=592 karg=ffffa06c197b4400 uarg=0000000000000000 00000080:00200080:3.0:1789760748.566985:0:5508:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908) cmd=c00866a4 arg=7ffdb86da240 00800000:00000080:3.0:1789760748.566988:0:5508:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06c18a1e000: cmd=c00866a4 len=592 karg=ffffa06c197b4400 uarg=0000000000000000 00000080:00200080:3.0:1789760748.566992:0:5508:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908) cmd=c00866a4 arg=7ffdb86da240 00800000:00000080:3.0:1789760748.566995:0:5508:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06c18a1e000: cmd=c00866a4 len=592 karg=ffffa06c197b4400 uarg=0000000000000000 00000080:00200080:3.0:1789760748.566998:0:5508:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908) cmd=c00866a4 arg=7ffdb86da240 00800000:00000080:3.0:1789760748.567001:0:5508:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06c18a1e000: cmd=c00866a4 len=592 karg=ffffa06c197b4400 uarg=0000000000000000 00000080:00200080:3.0:1789760748.567005:0:5508:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908) cmd=c00866a4 arg=7ffdb86da240 00800000:00000080:3.0:1789760748.567009:0:5508:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06c18a1e000: cmd=c00866a4 len=592 karg=ffffa06c197b4400 uarg=0000000000000000 00000080:00200080:3.0:1789760748.567012:0:5508:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908) cmd=c00866a4 arg=7ffdb86da240 00800000:00000080:3.0:1789760748.567016:0:5508:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06c18a1e000: cmd=c00866a4 len=592 karg=ffffa06c197b4400 uarg=0000000000000000 00000080:00200080:3.0:1789760748.567021:0:5508:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908) cmd=c00866a4 arg=7ffdb86da240 00800000:00000080:3.0:1789760748.567025:0:5508:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06c18a1e000: cmd=c00866a4 len=592 karg=ffffa06c197b4400 uarg=0000000000000000 00000080:00200080:3.0:1789760748.567029:0:5508:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908) cmd=c00866a4 arg=7ffdb86da240 00800000:00000080:3.0:1789760748.567033:0:5508:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06c18a1e000: cmd=c00866a4 len=592 karg=ffffa06c197b4400 uarg=0000000000000000 00000080:00200080:3.0:1789760748.567038:0:5508:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908) cmd=c00866a4 arg=7ffdb86da240 00800000:00000080:3.0:1789760748.567041:0:5508:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06c18a1e000: cmd=c00866a4 len=592 karg=ffffa06c197b4400 uarg=0000000000000000 00000080:00200080:3.0:1789760748.567045:0:5508:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908) cmd=c00866a4 arg=7ffdb86da240 00800000:00000080:3.0:1789760748.567049:0:5508:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06c18a1e000: cmd=c00866a4 len=592 karg=ffffa06c197b4400 uarg=0000000000000000 00000080:00200080:3.0:1789760748.567053:0:5508:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908) cmd=c00866a4 arg=7ffdb86da240 00800000:00000080:3.0:1789760748.567057:0:5508:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06c18a1e000: cmd=c00866a4 len=592 karg=ffffa06c197b4400 uarg=0000000000000000 00000080:00200080:3.0:1789760748.567068:0:5508:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908) cmd=c00866a4 arg=7ffdb86da240 00800000:00000080:3.0:1789760748.567084:0:5508:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06c18a1e000: cmd=c00866a4 len=592 karg=ffffa06c197b4400 uarg=0000000000000000 00000080:00200080:3.0:1789760748.567089:0:5508:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908) cmd=c00866a4 arg=7ffdb86da240 00800000:00000080:3.0:1789760748.567094:0:5508:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06c18a1e000: cmd=c00866a4 len=592 karg=ffffa06c197b4400 uarg=0000000000000000 00000080:00200080:3.0:1789760748.567098:0:5508:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908) cmd=c00866a4 arg=7ffdb86da240 00800000:00000080:3.0:1789760748.567101:0:5508:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06c18a1e000: cmd=c00866a4 len=592 karg=ffffa06c197b4400 uarg=0000000000000000 00000080:00200080:3.0:1789760748.567105:0:5508:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908) cmd=c00866a4 arg=7ffdb86da240 00800000:00000080:3.0:1789760748.567109:0:5508:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06c18a1e000: cmd=c00866a4 len=592 karg=ffffa06c197b4400 uarg=0000000000000000 00000080:00200080:3.0:1789760748.567112:0:5508:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908) cmd=c00866a4 arg=7ffdb86da240 00800000:00000080:3.0:1789760748.567116:0:5508:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06c18a1e000: cmd=c00866a4 len=592 karg=ffffa06c197b4400 uarg=0000000000000000 00000080:00200080:3.0:1789760748.567119:0:5508:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908) cmd=c00866a4 arg=7ffdb86da240 00800000:00000080:3.0:1789760748.567122:0:5508:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06c18a1e000: cmd=c00866a4 len=592 karg=ffffa06c197b4400 uarg=0000000000000000 00000080:00200080:3.0:1789760748.567126:0:5508:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908) cmd=c00866a4 arg=7ffdb86da240 00800000:00000080:3.0:1789760748.567129:0:5508:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06c18a1e000: cmd=c00866a4 len=592 karg=ffffa06c197b4400 uarg=0000000000000000 00000080:00200080:3.0:1789760748.567133:0:5508:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908) cmd=c00866a4 arg=7ffdb86da240 00800000:00000080:3.0:1789760748.567136:0:5508:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06c18a1e000: cmd=c00866a4 len=592 karg=ffffa06c197b4400 uarg=0000000000000000 00000080:00200080:3.0:1789760748.567140:0:5508:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908) cmd=c00866a4 arg=7ffdb86da240 00800000:00000080:3.0:1789760748.567143:0:5508:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06c18a1e000: cmd=c00866a4 len=592 karg=ffffa06c197b4400 uarg=0000000000000000 00000080:00200080:3.0:1789760748.567147:0:5508:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908) cmd=c00866a4 arg=7ffdb86da240 00800000:00000080:3.0:1789760748.567151:0:5508:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06c18a1e000: cmd=c00866a4 len=592 karg=ffffa06c197b4400 uarg=0000000000000000 00000080:00200080:3.0:1789760748.567154:0:5508:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908) cmd=c00866a4 arg=7ffdb86da240 00800000:00000080:3.0:1789760748.567158:0:5508:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06c18a1e000: cmd=c00866a4 len=592 karg=ffffa06c197b4400 uarg=0000000000000000 00000080:00200080:3.0:1789760748.567162:0:5508:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908) cmd=c00866a4 arg=7ffdb86da240 00800000:00000080:3.0:1789760748.567165:0:5508:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06c18a1e000: cmd=c00866a4 len=592 karg=ffffa06c197b4400 uarg=0000000000000000 00000080:00200080:3.0:1789760748.567168:0:5508:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908) cmd=c00866a4 arg=7ffdb86da240 00800000:00000080:3.0:1789760748.567179:0:5508:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06c18a1e000: cmd=c00866a4 len=592 karg=ffffa06c197b4400 uarg=0000000000000000 00000080:00200080:3.0:1789760748.567185:0:5508:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908) cmd=c00866a4 arg=7ffdb86da240 00800000:00000080:3.0:1789760748.567190:0:5508:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06c18a1e000: cmd=c00866a4 len=592 karg=ffffa06c197b4400 uarg=0000000000000000 00000080:00200080:3.0:1789760748.567194:0:5508:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908) cmd=c00866a4 arg=7ffdb86da240 00800000:00000080:3.0:1789760748.567198:0:5508:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06c18a1e000: cmd=c00866a4 len=592 karg=ffffa06c197b4400 uarg=0000000000000000 00000080:00200080:3.0:1789760748.567203:0:5508:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908) cmd=c00866a4 arg=7ffdb86da240 00800000:00000080:3.0:1789760748.567207:0:5508:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06c18a1e000: cmd=c00866a4 len=592 karg=ffffa06c197b4400 uarg=0000000000000000 00000080:00200080:3.0:1789760748.567211:0:5508:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908) cmd=c00866a4 arg=7ffdb86da240 00020000:00000080:3.0:1789760748.567217:0:5508:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06c18a1e000: cmd=c00866a4 len=592 karg=ffffa06c197b4400 uarg=0000000000000000 00020000:00000004:3.0:1789760748.567225:0:5508:0:(obd_class.h:1090:obd_statfs()) lustre-OST0000-osc-ffffa06c18a1e000: age -816, max_age 186 00000100:00100000:3.0:1789760748.567262:0:5508:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872ea00 pname:cluuid:pid:xid:nid:opc:job lfs:734aaee5-e21a-40f9-9d4e-90e3c76738d2:5508:1876700024540416:192.168.203.107@tcp:13: 00000100:00100000:3.0:1789760748.568314:0:5508:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c08e7b9c0 going to sleep for 11 seconds 00010000:00080000:2.0:1789760748.580122:0:5508:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61549) req@ffffa06c1872ea00 x1876700024540416/t0(0) o13->lustre-OST0000-osc-ffffa06c18a1e000@192.168.203.107@tcp:7/4 lens 224/368 e 0 to 0 dl 1789760764 ref 2 fl Rpc:RQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789760748.580142:0:5508:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872ea00 pname:cluuid:pid:xid:nid:opc:job lfs:734aaee5-e21a-40f9-9d4e-90e3c76738d2:5508:1876700024540416:192.168.203.107@tcp:13: 00020000:00000004:2.0:1789760748.580166:0:5508:0:(obd_class.h:1116:obd_statfs()) lustre-OST0000-osc-ffffa06c18a1e000: update ffffa06c08e31e08 cache blocks 901403/958279 objects 261841/262144 00000080:00200080:2.0:1789760748.580291:0:5508:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908) cmd=c00866a4 arg=7ffdb86da240 00020000:00000080:2.0:1789760748.580309:0:5508:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06c18a1e000: cmd=c00866a4 len=592 karg=ffffa06c09033000 uarg=0000000000000000 00020000:00000004:2.0:1789760748.580316:0:5508:0:(obd_class.h:1090:obd_statfs()) lustre-OST0001-osc-ffffa06c18a1e000: age -816, max_age 186 00000100:00100000:2.0:1789760748.580414:0:5508:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51500 pname:cluuid:pid:xid:nid:opc:job lfs:734aaee5-e21a-40f9-9d4e-90e3c76738d2:5508:1876700024540544:192.168.203.107@tcp:13: 00000100:00100000:2.0:1789760748.580611:0:5508:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c10a16a80 going to sleep for 11 seconds 00010000:00080000:2.0:1789760748.588444:0:5508:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61549) req@ffffa06c07e51500 x1876700024540544/t0(0) o13->lustre-OST0001-osc-ffffa06c18a1e000@192.168.203.107@tcp:7/4 lens 224/368 e 0 to 0 dl 1789760764 ref 2 fl Rpc:RQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789760748.588464:0:5508:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51500 pname:cluuid:pid:xid:nid:opc:job lfs:734aaee5-e21a-40f9-9d4e-90e3c76738d2:5508:1876700024540544:192.168.203.107@tcp:13: 00020000:00000004:2.0:1789760748.588503:0:5508:0:(obd_class.h:1116:obd_statfs()) lustre-OST0001-osc-ffffa06c18a1e000: update ffffa06c08e30450 cache blocks 901403/958279 objects 261841/262144 00000080:00200080:2.0:1789760748.588689:0:5508:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908) cmd=c00866a4 arg=7ffdb86da240 00020000:00000080:2.0:1789760748.588707:0:5508:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06c18a1e000: cmd=c00866a4 len=592 karg=ffffa06c09033000 uarg=0000000000000000 00000080:00200080:2.0:1789760748.588714:0:5508:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908) cmd=c00866a4 arg=7ffdb86da240 00020000:00000080:2.0:1789760748.588719:0:5508:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06c18a1e000: cmd=c00866a4 len=592 karg=ffffa06c09033000 uarg=0000000000000000 00000080:00200080:2.0:1789760748.588723:0:5508:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908) cmd=c00866a4 arg=7ffdb86da240 00020000:00000080:2.0:1789760748.588727:0:5508:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06c18a1e000: cmd=c00866a4 len=592 karg=ffffa06c09033000 uarg=0000000000000000 00000080:00200080:2.0:1789760748.588731:0:5508:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908) cmd=c00866a4 arg=7ffdb86da240 00020000:00000080:2.0:1789760748.588735:0:5508:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06c18a1e000: cmd=c00866a4 len=592 karg=ffffa06c09033000 uarg=0000000000000000 00000080:00200080:2.0:1789760748.588739:0:5508:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908) cmd=c00866a4 arg=7ffdb86da240 00020000:00000080:2.0:1789760748.588743:0:5508:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06c18a1e000: cmd=c00866a4 len=592 karg=ffffa06c09033000 uarg=0000000000000000 00000080:00200080:2.0:1789760748.588747:0:5508:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908) cmd=c00866a4 arg=7ffdb86da240 00020000:00000080:2.0:1789760748.588750:0:5508:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06c18a1e000: cmd=c00866a4 len=592 karg=ffffa06c09033000 uarg=0000000000000000 00000080:00200080:2.0:1789760748.588754:0:5508:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908) cmd=c00866a4 arg=7ffdb86da240 00020000:00000080:2.0:1789760748.588758:0:5508:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06c18a1e000: cmd=c00866a4 len=592 karg=ffffa06c09033000 uarg=0000000000000000 00000080:00200080:2.0:1789760748.588762:0:5508:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908) cmd=c00866a4 arg=7ffdb86da240 00020000:00000080:2.0:1789760748.588766:0:5508:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06c18a1e000: cmd=c00866a4 len=592 karg=ffffa06c09033000 uarg=0000000000000000 00000080:00200080:2.0:1789760748.588770:0:5508:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908) cmd=c00866a4 arg=7ffdb86da240 00020000:00000080:2.0:1789760748.588774:0:5508:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06c18a1e000: cmd=c00866a4 len=592 karg=ffffa06c09033000 uarg=0000000000000000 00000080:00200080:2.0:1789760748.588778:0:5508:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908) cmd=c00866a4 arg=7ffdb86da240 00020000:00000080:2.0:1789760748.588782:0:5508:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06c18a1e000: cmd=c00866a4 len=592 karg=ffffa06c09033000 uarg=0000000000000000 00000080:00200080:2.0:1789760748.588786:0:5508:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908) cmd=c00866a4 arg=7ffdb86da240 00020000:00000080:2.0:1789760748.588790:0:5508:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06c18a1e000: cmd=c00866a4 len=592 karg=ffffa06c09033000 uarg=0000000000000000 00000080:00200080:2.0:1789760748.588794:0:5508:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908) cmd=c00866a4 arg=7ffdb86da240 00020000:00000080:2.0:1789760748.588798:0:5508:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06c18a1e000: cmd=c00866a4 len=592 karg=ffffa06c09033000 uarg=0000000000000000 00000080:00200080:2.0:1789760748.588810:0:5508:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908) cmd=c00866a4 arg=7ffdb86da240 00020000:00000080:2.0:1789760748.588814:0:5508:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06c18a1e000: cmd=c00866a4 len=592 karg=ffffa06c09033000 uarg=0000000000000000 00000080:00200080:2.0:1789760748.588818:0:5508:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908) cmd=c00866a4 arg=7ffdb86da240 00020000:00000080:2.0:1789760748.588821:0:5508:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06c18a1e000: cmd=c00866a4 len=592 karg=ffffa06c09033000 uarg=0000000000000000 00000080:00200080:2.0:1789760748.588824:0:5508:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908) cmd=c00866a4 arg=7ffdb86da240 00020000:00000080:2.0:1789760748.588827:0:5508:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06c18a1e000: cmd=c00866a4 len=592 karg=ffffa06c09033000 uarg=0000000000000000 00000080:00200080:2.0:1789760748.588830:0:5508:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908) cmd=c00866a4 arg=7ffdb86da240 00020000:00000080:2.0:1789760748.588833:0:5508:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06c18a1e000: cmd=c00866a4 len=592 karg=ffffa06c09033000 uarg=0000000000000000 00000080:00200080:2.0:1789760748.588837:0:5508:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908) cmd=c00866a4 arg=7ffdb86da240 00020000:00000080:2.0:1789760748.588840:0:5508:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06c18a1e000: cmd=c00866a4 len=592 karg=ffffa06c09033000 uarg=0000000000000000 00000080:00200080:2.0:1789760748.588862:0:5508:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908) cmd=c00866a4 arg=7ffdb86da240 00020000:00000080:2.0:1789760748.588868:0:5508:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06c18a1e000: cmd=c00866a4 len=592 karg=ffffa06c09033000 uarg=0000000000000000 00000080:00200080:2.0:1789760748.588873:0:5508:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908) cmd=c00866a4 arg=7ffdb86da240 00020000:00000080:2.0:1789760748.588876:0:5508:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06c18a1e000: cmd=c00866a4 len=592 karg=ffffa06c09033000 uarg=0000000000000000 00000080:00200080:2.0:1789760748.588880:0:5508:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908) cmd=c00866a4 arg=7ffdb86da240 00020000:00000080:2.0:1789760748.588883:0:5508:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06c18a1e000: cmd=c00866a4 len=592 karg=ffffa06c09033000 uarg=0000000000000000 00000080:00200080:2.0:1789760748.588886:0:5508:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908) cmd=c00866a4 arg=7ffdb86da240 00020000:00000080:2.0:1789760748.588889:0:5508:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06c18a1e000: cmd=c00866a4 len=592 karg=ffffa06c09033000 uarg=0000000000000000 00000080:00200080:2.0:1789760748.588893:0:5508:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908) cmd=c00866a4 arg=7ffdb86da240 00020000:00000080:2.0:1789760748.588896:0:5508:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06c18a1e000: cmd=c00866a4 len=592 karg=ffffa06c09033000 uarg=0000000000000000 00000080:00200080:2.0:1789760748.588899:0:5508:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908) cmd=c00866a4 arg=7ffdb86da240 00020000:00000080:2.0:1789760748.588902:0:5508:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06c18a1e000: cmd=c00866a4 len=592 karg=ffffa06c09033000 uarg=0000000000000000 00000080:00200080:2.0:1789760748.588905:0:5508:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908) cmd=c00866a4 arg=7ffdb86da240 00020000:00000080:2.0:1789760748.588908:0:5508:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06c18a1e000: cmd=c00866a4 len=592 karg=ffffa06c09033000 uarg=0000000000000000 00000080:00200080:2.0:1789760748.588912:0:5508:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908) cmd=c00866a4 arg=7ffdb86da240 00020000:00000080:2.0:1789760748.588922:0:5508:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06c18a1e000: cmd=c00866a4 len=592 karg=ffffa06c09033000 uarg=0000000000000000 00000080:00200080:2.0:1789760748.588927:0:5508:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908) cmd=c00866a4 arg=7ffdb86da240 00020000:00000080:2.0:1789760748.588930:0:5508:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06c18a1e000: cmd=c00866a4 len=592 karg=ffffa06c09033000 uarg=0000000000000000 00000080:00200080:2.0:1789760748.588934:0:5508:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908) cmd=c00866a4 arg=7ffdb86da240 00020000:00000080:2.0:1789760748.588937:0:5508:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06c18a1e000: cmd=c00866a4 len=592 karg=ffffa06c09033000 uarg=0000000000000000 00000080:00200080:2.0:1789760748.588941:0:5508:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908) cmd=c00866a4 arg=7ffdb86da240 00020000:00000080:2.0:1789760748.588944:0:5508:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06c18a1e000: cmd=c00866a4 len=592 karg=ffffa06c09033000 uarg=0000000000000000 00000080:00200080:2.0:1789760748.588948:0:5508:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908) cmd=c00866a4 arg=7ffdb86da240 00020000:00000080:2.0:1789760748.588951:0:5508:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06c18a1e000: cmd=c00866a4 len=592 karg=ffffa06c09033000 uarg=0000000000000000 00000080:00200080:2.0:1789760748.588955:0:5508:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908) cmd=c00866a4 arg=7ffdb86da240 00020000:00000080:2.0:1789760748.588958:0:5508:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06c18a1e000: cmd=c00866a4 len=592 karg=ffffa06c09033000 uarg=0000000000000000 00000080:00200080:2.0:1789760748.588962:0:5508:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908) cmd=c00866a4 arg=7ffdb86da240 00020000:00000080:2.0:1789760748.588966:0:5508:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06c18a1e000: cmd=c00866a4 len=592 karg=ffffa06c09033000 uarg=0000000000000000 00000080:00200080:2.0:1789760748.588971:0:5508:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908) cmd=c00866a4 arg=7ffdb86da240 00020000:00000080:2.0:1789760748.588974:0:5508:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06c18a1e000: cmd=c00866a4 len=592 karg=ffffa06c09033000 uarg=0000000000000000 00000080:00200080:2.0:1789760748.588978:0:5508:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908) cmd=c00866a4 arg=7ffdb86da240 00020000:00000080:2.0:1789760748.588982:0:5508:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06c18a1e000: cmd=c00866a4 len=592 karg=ffffa06c09033000 uarg=0000000000000000 00000080:00200080:2.0:1789760748.588986:0:5508:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908) cmd=c00866a4 arg=7ffdb86da240 00020000:00000080:2.0:1789760748.588990:0:5508:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06c18a1e000: cmd=c00866a4 len=592 karg=ffffa06c09033000 uarg=0000000000000000 00000080:00200080:2.0:1789760748.588993:0:5508:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908) cmd=c00866a4 arg=7ffdb86da240 00020000:00000080:2.0:1789760748.588996:0:5508:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06c18a1e000: cmd=c00866a4 len=592 karg=ffffa06c09033000 uarg=0000000000000000 00000080:00200080:2.0:1789760748.589000:0:5508:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908) cmd=c00866a4 arg=7ffdb86da240 00020000:00000080:2.0:1789760748.589003:0:5508:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06c18a1e000: cmd=c00866a4 len=592 karg=ffffa06c09033000 uarg=0000000000000000 00000080:00200080:2.0:1789760748.589007:0:5508:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908) cmd=c00866a4 arg=7ffdb86da240 00020000:00000080:2.0:1789760748.589010:0:5508:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06c18a1e000: cmd=c00866a4 len=592 karg=ffffa06c09033000 uarg=0000000000000000 00000080:00200080:2.0:1789760748.589022:0:5508:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908) cmd=c00866a4 arg=7ffdb86da240 00020000:00000080:2.0:1789760748.589027:0:5508:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06c18a1e000: cmd=c00866a4 len=592 karg=ffffa06c09033000 uarg=0000000000000000 00000080:00200080:2.0:1789760748.589033:0:5508:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908) cmd=c00866a4 arg=7ffdb86da240 00020000:00000080:2.0:1789760748.589037:0:5508:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06c18a1e000: cmd=c00866a4 len=592 karg=ffffa06c09033000 uarg=0000000000000000 00000080:00200080:2.0:1789760748.589042:0:5508:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908) cmd=c00866a4 arg=7ffdb86da240 00020000:00000080:2.0:1789760748.589045:0:5508:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06c18a1e000: cmd=c00866a4 len=592 karg=ffffa06c09033000 uarg=0000000000000000 00000080:00200080:2.0:1789760748.589049:0:5508:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908) cmd=c00866a4 arg=7ffdb86da240 00020000:00000080:2.0:1789760748.589053:0:5508:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06c18a1e000: cmd=c00866a4 len=592 karg=ffffa06c09033000 uarg=0000000000000000 00000080:00200080:2.0:1789760748.589056:0:5508:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908) cmd=c00866a4 arg=7ffdb86da240 00020000:00000080:2.0:1789760748.589059:0:5508:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06c18a1e000: cmd=c00866a4 len=592 karg=ffffa06c09033000 uarg=0000000000000000 00000080:00200080:2.0:1789760748.589063:0:5508:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908) cmd=c00866a4 arg=7ffdb86da240 00020000:00000080:2.0:1789760748.589066:0:5508:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06c18a1e000: cmd=c00866a4 len=592 karg=ffffa06c09033000 uarg=0000000000000000 00000080:00200080:2.0:1789760748.589083:0:5508:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908) cmd=c00866a4 arg=7ffdb86da240 00020000:00000080:2.0:1789760748.589087:0:5508:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06c18a1e000: cmd=c00866a4 len=592 karg=ffffa06c09033000 uarg=0000000000000000 00000080:00200080:2.0:1789760748.589092:0:5508:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908) cmd=c00866a4 arg=7ffdb86da240 00020000:00000080:2.0:1789760748.589096:0:5508:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06c18a1e000: cmd=c00866a4 len=592 karg=ffffa06c09033000 uarg=0000000000000000 00000080:00200080:2.0:1789760748.589100:0:5508:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908) cmd=c00866a4 arg=7ffdb86da240 00020000:00000080:2.0:1789760748.589102:0:5508:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06c18a1e000: cmd=c00866a4 len=592 karg=ffffa06c09033000 uarg=0000000000000000 00000080:00200080:2.0:1789760748.589106:0:5508:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908) cmd=c00866a4 arg=7ffdb86da240 00020000:00000080:2.0:1789760748.589109:0:5508:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06c18a1e000: cmd=c00866a4 len=592 karg=ffffa06c09033000 uarg=0000000000000000 00000080:00200080:2.0:1789760748.589113:0:5508:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908) cmd=c00866a4 arg=7ffdb86da240 00020000:00000080:2.0:1789760748.589116:0:5508:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06c18a1e000: cmd=c00866a4 len=592 karg=ffffa06c09033000 uarg=0000000000000000 00000080:00200080:2.0:1789760748.589121:0:5508:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908) cmd=c00866a4 arg=7ffdb86da240 00020000:00000080:2.0:1789760748.589125:0:5508:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06c18a1e000: cmd=c00866a4 len=592 karg=ffffa06c09033000 uarg=0000000000000000 00000080:00200080:2.0:1789760748.589130:0:5508:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908) cmd=c00866a4 arg=7ffdb86da240 00020000:00000080:2.0:1789760748.589140:0:5508:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06c18a1e000: cmd=c00866a4 len=592 karg=ffffa06c09033000 uarg=0000000000000000 00000080:00200080:2.0:1789760748.589146:0:5508:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908) cmd=c00866a4 arg=7ffdb86da240 00020000:00000080:2.0:1789760748.589149:0:5508:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06c18a1e000: cmd=c00866a4 len=592 karg=ffffa06c09033000 uarg=0000000000000000 00000080:00200080:2.0:1789760748.589152:0:5508:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908) cmd=c00866a4 arg=7ffdb86da240 00020000:00000080:2.0:1789760748.589156:0:5508:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06c18a1e000: cmd=c00866a4 len=592 karg=ffffa06c09033000 uarg=0000000000000000 00000080:00200080:2.0:1789760748.589161:0:5508:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908) cmd=c00866a4 arg=7ffdb86da240 00020000:00000080:2.0:1789760748.589165:0:5508:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06c18a1e000: cmd=c00866a4 len=592 karg=ffffa06c09033000 uarg=0000000000000000 00000080:00200080:2.0:1789760748.589169:0:5508:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908) cmd=c00866a4 arg=7ffdb86da240 00020000:00000080:2.0:1789760748.589173:0:5508:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06c18a1e000: cmd=c00866a4 len=592 karg=ffffa06c09033000 uarg=0000000000000000 00000080:00200080:2.0:1789760748.589176:0:5508:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908) cmd=c00866a4 arg=7ffdb86da240 00020000:00000080:2.0:1789760748.589179:0:5508:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06c18a1e000: cmd=c00866a4 len=592 karg=ffffa06c09033000 uarg=0000000000000000 00000080:00200080:2.0:1789760748.589182:0:5508:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908) cmd=c00866a4 arg=7ffdb86da240 00020000:00000080:2.0:1789760748.589185:0:5508:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06c18a1e000: cmd=c00866a4 len=592 karg=ffffa06c09033000 uarg=0000000000000000 00000080:00200080:2.0:1789760748.589188:0:5508:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908) cmd=c00866a4 arg=7ffdb86da240 00020000:00000080:2.0:1789760748.589191:0:5508:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06c18a1e000: cmd=c00866a4 len=592 karg=ffffa06c09033000 uarg=0000000000000000 00000080:00200080:2.0:1789760748.589195:0:5508:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908) cmd=c00866a4 arg=7ffdb86da240 00020000:00000080:2.0:1789760748.589198:0:5508:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06c18a1e000: cmd=c00866a4 len=592 karg=ffffa06c09033000 uarg=0000000000000000 00000080:00200080:2.0:1789760748.589202:0:5508:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908) cmd=c00866a4 arg=7ffdb86da240 00020000:00000080:2.0:1789760748.589205:0:5508:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06c18a1e000: cmd=c00866a4 len=592 karg=ffffa06c09033000 uarg=0000000000000000 00000080:00200080:2.0:1789760748.589208:0:5508:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908) cmd=c00866a4 arg=7ffdb86da240 00020000:00000080:2.0:1789760748.589211:0:5508:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06c18a1e000: cmd=c00866a4 len=592 karg=ffffa06c09033000 uarg=0000000000000000 00000080:00200080:2.0:1789760748.589215:0:5508:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908) cmd=c00866a4 arg=7ffdb86da240 00020000:00000080:2.0:1789760748.589218:0:5508:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06c18a1e000: cmd=c00866a4 len=592 karg=ffffa06c09033000 uarg=0000000000000000 00000080:00200080:2.0:1789760748.589221:0:5508:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908) cmd=c00866a4 arg=7ffdb86da240 00020000:00000080:2.0:1789760748.589225:0:5508:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06c18a1e000: cmd=c00866a4 len=592 karg=ffffa06c09033000 uarg=0000000000000000 00000080:00200080:2.0:1789760748.589236:0:5508:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908) cmd=c00866a4 arg=7ffdb86da240 00020000:00000080:2.0:1789760748.589241:0:5508:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06c18a1e000: cmd=c00866a4 len=592 karg=ffffa06c09033000 uarg=0000000000000000 00000080:00200008:2.0:1789760748.589255:0:5508:0:(file.c:479:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffffa06c03518908), flags 100000 00010000:00010000:2.0:1789760749.175002:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789760750.199190:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000001:02000400:3.0:1789760750.635301:0:5697:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: Using TIMEOUT=20 00010000:00010000:3.0:1789760751.223252:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789760751.479026:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2344:1876700024540672:192.168.203.107@tcp:400: 00000100:00100000:2.0:1789760751.479154:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2344:1876700024540928:192.168.203.107@tcp:400: 00000100:00100000:2.0:1789760751.479189:0:2344:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [0->1] 00000100:00100000:2.0:1789760751.479195:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2344:1876700024540800:192.168.203.107@tcp:400: 00000100:00100000:2.0:1789760751.479217:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e53800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2344:1876700024541056:192.168.203.107@tcp:400: 00000100:00100000:2.0:1789760751.488749:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2344:1876700024540800:192.168.203.107@tcp:400: 00010000:00080000:2.0:1789760751.488771:0:2344:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61549) req@ffffa06c07e53800 x1876700024541056/t0(0) o400->lustre-OST0001-osc-ffffa06c18a1e000@192.168.203.107@tcp:28/4 lens 224/224 e 0 to 0 dl 1789760767 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789760751.488782:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e53800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2344:1876700024541056:192.168.203.107@tcp:400: 00000100:00100000:2.0:1789760751.488791:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2344:1876700024540672:192.168.203.107@tcp:400: 00010000:00080000:2.0:1789760751.488800:0:2344:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61549) req@ffffa06c07e51880 x1876700024540928/t0(0) o400->lustre-OST0000-osc-ffffa06c18a1e000@192.168.203.107@tcp:28/4 lens 224/224 e 0 to 0 dl 1789760767 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789760751.488809:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2344:1876700024540928:192.168.203.107@tcp:400: 00010000:00010000:1.0:1789760752.247282:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789760753.271181:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789760753.859041:0:2334:0:(events.c:340:request_in_callback()) peer: 12345-192.168.203.107@tcp (source: 12345-192.168.203.107@tcp) 00000100:00100000:0.0:1789760753.859107:0:2334:0:(events.c:340:request_in_callback()) peer: 12345-192.168.203.107@tcp (source: 12345-192.168.203.107@tcp) 00000100:00100000:1.0:1789760753.859482:0:5417:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1876700049727488 00000100:00100000:1.0:1789760753.859495:0:5417:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1876700049727488 00000100:00100000:1.0:1789760753.859553:0:5417:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1876700049727616 00000100:00100000:1.0:1789760753.859556:0:5417:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1876700049727616 00000100:00100000:1.0:1789760753.859574:0:5417:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.107@tcp, seq: 0 00000100:00100000:1.0:1789760753.859584:0:5417:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffffa06c091d1880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:02bc4b43-5e6f-4d7e-9224-80f5c789104f+4:8812:x1876700049727488:12345-192.168.203.107@tcp:104: 00010000:00010000:1.0:1789760753.859605:0:5417:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: MGC192.168.203.107@tcp lock: ffffa06c01513000/0x59bc99bb6194805f lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x1400000000000 nid: local remote: 0x1daf3002fe9e0867 expref: -99 pid: 5415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1789760753.859707:0:5417:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1789760753.859714:0:5417:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffffa06c091d1880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:02bc4b43-5e6f-4d7e-9224-80f5c789104f+4:8812:x1876700049727488:12345-192.168.203.107@tcp:104: Request processed in 131us (699us total) trans 0 rc 0/0 00000100:00100000:1.0:1789760753.859721:0:5417:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.107@tcp, seq: 0 00000100:00100000:1.0:1789760753.859741:0:5417:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.107@tcp, seq: 1 00000100:00100000:1.0:1789760753.859748:0:5417:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffffa06c091d3800 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:02bc4b43-5e6f-4d7e-9224-80f5c789104f+3:8812:x1876700049727616:12345-192.168.203.107@tcp:104: 00010000:00010000:1.0:1789760753.859775:0:5417:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: MGC192.168.203.107@tcp lock: ffffa06c01512c00/0x59bc99bb61948058 lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x1400000000000 nid: local remote: 0x1daf3002fe9e0860 expref: -99 pid: 5415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1789760753.859808:0:5417:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to priority blp list, 2 blwis in pool 00000100:00100000:1.0:1789760753.859814:0:5417:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffffa06c091d3800 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:02bc4b43-5e6f-4d7e-9224-80f5c789104f+3:8812:x1876700049727616:12345-192.168.203.107@tcp:104: Request processed in 67us (707us total) trans 0 rc 0/0 00000100:00100000:1.0:1789760753.859819:0:5417:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.107@tcp, seq: 1 00010000:00010000:2.0:1789760753.860191:0:5428:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:2.0:1789760753.860200:0:5428:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.203.107@tcp lock: ffffa06c01513000/0x59bc99bb6194805f lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x1400000000000 nid: local remote: 0x1daf3002fe9e0867 expref: -99 pid: 5415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1789760753.860209:0:5428:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffffa06c01513000 already unused, calling callback (ffffffffc13cfc20) 10000000:00010000:2.0:1789760753.860221:0:5428:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.203.107@tcp lock: ffffa06c01513000/0x59bc99bb6194805f lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x1400400000000 nid: local remote: 0x1daf3002fe9e0867 expref: -99 pid: 5415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1789760753.860231:0:5428:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.203.107@tcp lock: ffffa06c01513000/0x59bc99bb6194805f lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x1408400000000 nid: local remote: 0x1daf3002fe9e0867 expref: -99 pid: 5415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00010000:2.0:1789760753.860239:0:5428:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.203.107@tcp lock: ffffa06c01513000/0x59bc99bb6194805f lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x1409400000000 nid: local remote: 0x1daf3002fe9e0867 expref: -99 pid: 5415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:2.0:1789760753.860246:0:5428:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 00010000:00010000:2.0:1789760753.860300:0:5428:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: MGC192.168.203.107@tcp lock: ffffa06c01513000/0x59bc99bb6194805f lrc: 2/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x5c09400000020 nid: local remote: 0x1daf3002fe9e0867 expref: -99 pid: 5415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1789760753.860308:0:5428:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1789760753.860332:0:5428:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.203.107@tcp lock: ffffa06c01513000/0x59bc99bb6194805f lrc: 1/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x5c09400000020 nid: local remote: 0x1daf3002fe9e0867 expref: -99 pid: 5415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1789760753.860351:0:5428:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.203.107@tcp lock: ffffa06c01513000/0x59bc99bb6194805f lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x5c09400000020 nid: local remote: 0x1daf3002fe9e0867 expref: -99 pid: 5415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1789760753.860375:0:5428:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1789760753.860379:0:5428:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.203.107@tcp lock: ffffa06c01512c00/0x59bc99bb61948058 lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1400000000000 nid: local remote: 0x1daf3002fe9e0860 expref: -99 pid: 5415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1789760753.860385:0:5428:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffffa06c01512c00 already unused, calling callback (ffffffffc13cfc20) 10000000:00010000:2.0:1789760753.860388:0:5428:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.203.107@tcp lock: ffffa06c01512c00/0x59bc99bb61948058 lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1400400000000 nid: local remote: 0x1daf3002fe9e0860 expref: -99 pid: 5415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1789760753.860396:0:5428:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.203.107@tcp lock: ffffa06c01512c00/0x59bc99bb61948058 lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1408400000000 nid: local remote: 0x1daf3002fe9e0860 expref: -99 pid: 5415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00010000:2.0:1789760753.860414:0:5428:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.203.107@tcp lock: ffffa06c01512c00/0x59bc99bb61948058 lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1409400000000 nid: local remote: 0x1daf3002fe9e0860 expref: -99 pid: 5415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:2.0:1789760753.860420:0:5428:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 00010000:00010000:2.0:1789760753.860434:0:5428:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: MGC192.168.203.107@tcp lock: ffffa06c01512c00/0x59bc99bb61948058 lrc: 2/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5c09400000020 nid: local remote: 0x1daf3002fe9e0860 expref: -99 pid: 5415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1789760753.860442:0:5428:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1789760753.860447:0:5428:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.203.107@tcp lock: ffffa06c01512c00/0x59bc99bb61948058 lrc: 1/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5c09400000020 nid: local remote: 0x1daf3002fe9e0860 expref: -99 pid: 5415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1789760753.860455:0:5428:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.203.107@tcp lock: ffffa06c01512c00/0x59bc99bb61948058 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5c09400000020 nid: local remote: 0x1daf3002fe9e0860 expref: -99 pid: 5415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1789760753.860473:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789760753.860475:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789760753.860477:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789760753.860498:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2344:1876700024541184:192.168.203.107@tcp:103: 00010000:00010000:3.0:1789760753.860629:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789760753.860708:0:2343:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2343:1876700024541312:192.168.203.107@tcp:103: 00000100:00100000:3.0:1789760753.880311:0:2343:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2343:1876700024541312:192.168.203.107@tcp:103: 00000100:00100000:2.0:1789760753.882893:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2344:1876700024541184:192.168.203.107@tcp:103: 00010000:00010000:2.0:1789760754.295340:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 10000000:01000000:0.0:1789760755.192061:0:5431:0:(mgc_request.c:611:do_requeue()) updating log lustre-client 10000000:01000000:0.0:1789760755.192087:0:5431:0:(mgc_request.c:1802:mgc_process_log()) Process log lustre-client-ffffa06c18a1e000 from 39 10000000:01000000:0.0:1789760755.192092:0:5431:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-client (res 0x65727473756c) 00010000:00010000:0.0:1789760755.192145:0:5431:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.203.107@tcp lock: ffffa06c110ca000/0x59bc99bb619480c1 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 5431 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1789760755.192158:0:5431:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.203.107@tcp lock: ffffa06c110ca000/0x59bc99bb619480c1 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 5431 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1789760755.192170:0:5431:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.203.107@tcp lock: ffffa06c110ca000/0x59bc99bb619480c1 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 5431 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1789760755.192198:0:5431:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0380 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:02bc4b43-5e6f-4d7e-9224-80f5c789104f:5431:1876700024541440:192.168.203.107@tcp:101: 00000100:00100000:0.0:1789760755.192309:0:5431:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c1870c000 going to sleep for 11 seconds 00000100:00100000:1.0:1789760755.199610:0:5431:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0380 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:02bc4b43-5e6f-4d7e-9224-80f5c789104f:5431:1876700024541440:192.168.203.107@tcp:101: 00010000:00010000:1.0:1789760755.199646:0:5431:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: MGC192.168.203.107@tcp lock: ffffa06c110ca000/0x59bc99bb619480c1 lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x1daf3002fe9e08a6 expref: -99 pid: 5431 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1789760755.199660:0:5431:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: MGC192.168.203.107@tcp lock: ffffa06c110ca000/0x59bc99bb619480c1 lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x1daf3002fe9e08a6 expref: -99 pid: 5431 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1789760755.199747:0:5431:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2c700 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:02bc4b43-5e6f-4d7e-9224-80f5c789104f:5431:1876700024541568:192.168.203.107@tcp:501: 00000100:00100000:1.0:1789760755.199839:0:5431:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c0985df00 going to sleep for 11 seconds 00000100:00100000:1.0:1789760755.202049:0:5431:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2c700 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:02bc4b43-5e6f-4d7e-9224-80f5c789104f:5431:1876700024541568:192.168.203.107@tcp:501: 00000100:00100000:1.0:1789760755.202107:0:5431:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2c700 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:02bc4b43-5e6f-4d7e-9224-80f5c789104f:5431:1876700024541696:192.168.203.107@tcp:503: 00000100:00100000:1.0:1789760755.202241:0:5431:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c0985df00 going to sleep for 11 seconds 00000100:00100000:1.0:1789760755.206737:0:5431:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2c700 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:02bc4b43-5e6f-4d7e-9224-80f5c789104f:5431:1876700024541696:192.168.203.107@tcp:503: 00000100:00100000:1.0:1789760755.207241:0:5902:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2c700 pname:cluuid:pid:xid:nid:opc:job llog_process_th:02bc4b43-5e6f-4d7e-9224-80f5c789104f:5902:1876700024541824:192.168.203.107@tcp:502: 00000100:00100000:1.0:1789760755.207391:0:5902:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c0985dcc0 going to sleep for 11 seconds 00000100:00100000:1.0:1789760755.211793:0:5902:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2c700 pname:cluuid:pid:xid:nid:opc:job llog_process_th:02bc4b43-5e6f-4d7e-9224-80f5c789104f:5902:1876700024541824:192.168.203.107@tcp:502: 00000020:01000000:1.0:1789760755.211840:0:5902:0:(obd_config.c:1893:class_config_llog_handler()) Marker, inst_flg=0x0 mark_flg=0x1 00000020:00000080:1.0:1789760755.211929:0:5902:0:(obd_config.c:1472:class_process_config()) processing cmd: cf010 00000020:00000080:1.0:1789760755.211932:0:5902:0:(obd_config.c:1563:class_process_config()) marker 26 (0x1) lustre sys.jobid_var 00000020:00000080:1.0:1789760755.211938:0:5902:0:(obd_config.c:1472:class_process_config()) processing cmd: cf00f 00000100:00100000:1.0:1789760755.211978:0:5902:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2c700 pname:cluuid:pid:xid:nid:opc:job llog_process_th:02bc4b43-5e6f-4d7e-9224-80f5c789104f:5902:1876700024541952:192.168.203.107@tcp:502:llog_process_th.0 00000100:00100000:1.0:1789760755.212079:0:5902:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c0985dcc0 going to sleep for 11 seconds 00000100:00100000:1.0:1789760755.215819:0:5902:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2c700 pname:cluuid:pid:xid:nid:opc:job llog_process_th:02bc4b43-5e6f-4d7e-9224-80f5c789104f:5902:1876700024541952:192.168.203.107@tcp:502:llog_process_th.0 00000020:01000000:1.0:1789760755.215891:0:5902:0:(obd_config.c:1893:class_config_llog_handler()) Marker, inst_flg=0x2 mark_flg=0x2 00000020:00000080:1.0:1789760755.215900:0:5902:0:(obd_config.c:1472:class_process_config()) processing cmd: cf010 00000020:00000080:1.0:1789760755.215902:0:5902:0:(obd_config.c:1563:class_process_config()) marker 26 (0x2) lustre sys.jobid_var 00000040:00080000:1.0:1789760755.215907:0:5902:0:(llog.c:862:llog_process_thread()) stop processing plain [0xa:0x3:0x0] index 42 count 42 00000020:01000000:0.0:1789760755.216119:0:5431:0:(obd_config.c:2145:class_config_parse_llog()) Processed log lustre-client gen 39-41 (rc=0) 10000000:01000000:0.0:1789760755.216145:0:5431:0:(mgc_request.c:1879:mgc_process_log()) MGC192.168.203.107@tcp: configuration from log 'lustre-client' succeeded (0). 00010000:00010000:0.0:1789760755.216158:0:5431:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.203.107@tcp lock: ffffa06c110ca000/0x59bc99bb619480c1 lrc: 3/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x1daf3002fe9e08a6 expref: -99 pid: 5431 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1789760755.216169:0:5431:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: MGC192.168.203.107@tcp lock: ffffa06c110ca000/0x59bc99bb619480c1 lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x1daf3002fe9e08a6 expref: -99 pid: 5431 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:0.0:1789760755.216179:0:5431:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:01000000:0.0:1789760755.216182:0:5431:0:(mgc_request.c:1802:mgc_process_log()) Process log lustre-sptlrpc-ffffa06c2011cd28 from 1 10000000:01000000:0.0:1789760755.216185:0:5431:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00010000:00010000:0.0:1789760755.216216:0:5431:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.203.107@tcp lock: ffffa06c110cb600/0x59bc99bb619480c8 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 5431 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1789760755.216224:0:5431:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.203.107@tcp lock: ffffa06c110cb600/0x59bc99bb619480c8 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 5431 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1789760755.216234:0:5431:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.203.107@tcp lock: ffffa06c110cb600/0x59bc99bb619480c8 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 5431 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1789760755.216265:0:5431:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d1f80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:02bc4b43-5e6f-4d7e-9224-80f5c789104f:5431:1876700024542080:192.168.203.107@tcp:101:ll_cfg_requeue.0 00000100:00100000:0.0:1789760755.216350:0:5431:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c1870cb40 going to sleep for 11 seconds 00000100:00100000:1.0:1789760755.220468:0:5431:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d1f80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:02bc4b43-5e6f-4d7e-9224-80f5c789104f:5431:1876700024542080:192.168.203.107@tcp:101:ll_cfg_requeue.0 00010000:00010000:1.0:1789760755.220499:0:5431:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: MGC192.168.203.107@tcp lock: ffffa06c110cb600/0x59bc99bb619480c8 lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x11000000000000 nid: local remote: 0x1daf3002fe9e08ad expref: -99 pid: 5431 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1789760755.220513:0:5431:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: MGC192.168.203.107@tcp lock: ffffa06c110cb600/0x59bc99bb619480c8 lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x1000000000000 nid: local remote: 0x1daf3002fe9e08ad expref: -99 pid: 5431 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1789760755.220595:0:5431:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2c700 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:02bc4b43-5e6f-4d7e-9224-80f5c789104f:5431:1876700024542208:192.168.203.107@tcp:501:ll_cfg_requeue.0 00000100:00100000:1.0:1789760755.220677:0:5431:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c0985dcc0 going to sleep for 11 seconds 00000100:00100000:2.0:1789760755.225463:0:5431:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2c700 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:02bc4b43-5e6f-4d7e-9224-80f5c789104f:5431:1876700024542208:192.168.203.107@tcp:501:ll_cfg_requeue.0 10000000:01000000:2.0:1789760755.225509:0:5431:0:(mgc_request.c:1879:mgc_process_log()) MGC192.168.203.107@tcp: configuration from log 'lustre-sptlrpc' failed (-2). 00010000:00010000:2.0:1789760755.225530:0:5431:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.203.107@tcp lock: ffffa06c110cb600/0x59bc99bb619480c8 lrc: 3/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x11000000000000 nid: local remote: 0x1daf3002fe9e08ad expref: -99 pid: 5431 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1789760755.225542:0:5431:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: MGC192.168.203.107@tcp lock: ffffa06c110cb600/0x59bc99bb619480c8 lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x11000000000000 nid: local remote: 0x1daf3002fe9e08ad expref: -99 pid: 5431 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1789760755.319015:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789760756.344307:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789760756.599123:0:2343:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2343:1876700024542464:192.168.203.107@tcp:400:kworker.0 00000100:00100000:3.0:1789760756.599293:0:2343:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2343:1876700024542720:192.168.203.107@tcp:400:kworker.0 00000100:00100000:3.0:1789760756.599326:0:2343:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [1->0] 00000100:00100000:3.0:1789760756.599332:0:2343:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2343:1876700024542336:192.168.203.107@tcp:400:kworker.0 00000100:00100000:3.0:1789760756.599362:0:2343:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2343:1876700024542592:192.168.203.107@tcp:400:kworker.0 00000100:00100000:3.0:1789760756.603374:0:2343:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2343:1876700024542464:192.168.203.107@tcp:400:kworker.0 00010000:00080000:3.0:1789760756.603394:0:2343:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61549) req@ffffa06c1872f100 x1876700024542720/t0(0) o400->lustre-OST0001-osc-ffffa06c18a1e000@192.168.203.107@tcp:28/4 lens 224/224 e 0 to 0 dl 1789760772 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789760756.603457:0:2343:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2343:1876700024542720:192.168.203.107@tcp:400:kworker.0 00000100:00100000:3.0:1789760756.603484:0:2343:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2343:1876700024542336:192.168.203.107@tcp:400:kworker.0 00010000:00080000:3.0:1789760756.603495:0:2343:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61549) req@ffffa06c1872c700 x1876700024542592/t0(0) o400->lustre-OST0000-osc-ffffa06c18a1e000@192.168.203.107@tcp:28/4 lens 224/224 e 0 to 0 dl 1789760772 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789760756.603503:0:2343:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2343:1876700024542592:192.168.203.107@tcp:400:kworker.0 00000100:00100000:1.0:1789760756.825648:0:2334:0:(events.c:340:request_in_callback()) peer: 12345-192.168.203.107@tcp (source: 12345-192.168.203.107@tcp) 00000100:00100000:0.0:1789760756.825792:0:5417:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1876700049734528 00000100:00100000:0.0:1789760756.825803:0:5417:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1876700049734528 00000100:00100000:0.0:1789760756.825885:0:5417:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.107@tcp, seq: 2 00000100:00100000:0.0:1789760756.825896:0:5417:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffffa06c08e2f100 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:02bc4b43-5e6f-4d7e-9224-80f5c789104f+3:8870:x1876700049734528:12345-192.168.203.107@tcp:104: 00010000:00010000:0.0:1789760756.825912:0:5417:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: MGC192.168.203.107@tcp lock: ffffa06c42175600/0x59bc99bb61948090 lrc: 2/0,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x1400000000000 nid: local remote: 0x1daf3002fe9e0875 expref: -99 pid: 5415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1789760756.825993:0:5417:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1789760756.826000:0:5417:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffffa06c08e2f100 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:02bc4b43-5e6f-4d7e-9224-80f5c789104f+3:8870:x1876700049734528:12345-192.168.203.107@tcp:104: Request processed in 107us (363us total) trans 0 rc 0/0 00000100:00100000:0.0:1789760756.826017:0:5417:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.107@tcp, seq: 2 00010000:00010000:3.0:1789760756.826195:0:5427:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1789760756.826202:0:5427:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.203.107@tcp lock: ffffa06c42175600/0x59bc99bb61948090 lrc: 2/0,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x1400000000000 nid: local remote: 0x1daf3002fe9e0875 expref: -99 pid: 5415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1789760756.826210:0:5427:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffffa06c42175600 already unused, calling callback (ffffffffc13cfc20) 10000000:00010000:3.0:1789760756.826214:0:5427:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.203.107@tcp lock: ffffa06c42175600/0x59bc99bb61948090 lrc: 2/0,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x1400400000000 nid: local remote: 0x1daf3002fe9e0875 expref: -99 pid: 5415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1789760756.826222:0:5427:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.203.107@tcp lock: ffffa06c42175600/0x59bc99bb61948090 lrc: 3/0,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x1408400000000 nid: local remote: 0x1daf3002fe9e0875 expref: -99 pid: 5415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00010000:3.0:1789760756.826230:0:5427:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.203.107@tcp lock: ffffa06c42175600/0x59bc99bb61948090 lrc: 3/0,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x1409400000000 nid: local remote: 0x1daf3002fe9e0875 expref: -99 pid: 5415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:3.0:1789760756.826236:0:5427:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x736d61726170:0x3:0x0].0x0 (params) 00010000:00010000:3.0:1789760756.826329:0:5427:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: MGC192.168.203.107@tcp lock: ffffa06c42175600/0x59bc99bb61948090 lrc: 2/0,0 mode: --/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x5c09400000020 nid: local remote: 0x1daf3002fe9e0875 expref: -99 pid: 5415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1789760756.826351:0:5427:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:3.0:1789760756.826374:0:5427:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.203.107@tcp lock: ffffa06c42175600/0x59bc99bb61948090 lrc: 1/0,0 mode: --/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x5c09400000020 nid: local remote: 0x1daf3002fe9e0875 expref: -99 pid: 5415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1789760756.826381:0:5427:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.203.107@tcp lock: ffffa06c42175600/0x59bc99bb61948090 lrc: 0/0,0 mode: --/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x5c09400000020 nid: local remote: 0x1daf3002fe9e0875 expref: -99 pid: 5415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1789760756.826402:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789760756.826404:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789760756.826405:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789760756.826482:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2344:1876700024542848:192.168.203.107@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1789760756.829081:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2344:1876700024542848:192.168.203.107@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1789760757.367283:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 10000000:01000000:2.0:1789760757.945790:0:5431:0:(mgc_request.c:611:do_requeue()) updating log params 10000000:01000000:2.0:1789760757.945801:0:5431:0:(mgc_request.c:1802:mgc_process_log()) Process log params-ffffa06c18a1e000 from 4 10000000:01000000:2.0:1789760757.945807:0:5431:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for params (res 0x736d61726170) 00010000:00010000:2.0:1789760757.945915:0:5431:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.203.107@tcp lock: ffffa06c01512e00/0x59bc99bb619480cf lrc: 3/1,0 mode: --/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 5431 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1789760757.945930:0:5431:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.203.107@tcp lock: ffffa06c01512e00/0x59bc99bb619480cf lrc: 3/1,0 mode: --/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 5431 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1789760757.945943:0:5431:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.203.107@tcp lock: ffffa06c01512e00/0x59bc99bb619480cf lrc: 3/1,0 mode: --/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 5431 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1789760757.945974:0:5431:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51880 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:02bc4b43-5e6f-4d7e-9224-80f5c789104f:5431:1876700024542976:192.168.203.107@tcp:101:ll_cfg_requeue.0 00000100:00100000:2.0:1789760757.946108:0:5431:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c056ba300 going to sleep for 11 seconds 00000100:00100000:2.0:1789760757.950715:0:5431:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51880 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:02bc4b43-5e6f-4d7e-9224-80f5c789104f:5431:1876700024542976:192.168.203.107@tcp:101:ll_cfg_requeue.0 00010000:00010000:2.0:1789760757.950759:0:5431:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: MGC192.168.203.107@tcp lock: ffffa06c01512e00/0x59bc99bb619480cf lrc: 4/1,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x1daf3002fe9e0939 expref: -99 pid: 5431 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1789760757.950777:0:5431:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: MGC192.168.203.107@tcp lock: ffffa06c01512e00/0x59bc99bb619480cf lrc: 4/1,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x1daf3002fe9e0939 expref: -99 pid: 5431 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1789760757.950937:0:5431:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51880 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:02bc4b43-5e6f-4d7e-9224-80f5c789104f:5431:1876700024543104:192.168.203.107@tcp:501:ll_cfg_requeue.0 00000100:00100000:2.0:1789760757.951053:0:5431:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c056ba300 going to sleep for 11 seconds 00000100:00100000:2.0:1789760757.955746:0:5431:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51880 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:02bc4b43-5e6f-4d7e-9224-80f5c789104f:5431:1876700024543104:192.168.203.107@tcp:501:ll_cfg_requeue.0 00000100:00100000:2.0:1789760757.955831:0:5431:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51880 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:02bc4b43-5e6f-4d7e-9224-80f5c789104f:5431:1876700024543232:192.168.203.107@tcp:503:ll_cfg_requeue.0 00000100:00100000:2.0:1789760757.956020:0:5431:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c056ba300 going to sleep for 11 seconds 00000100:00100000:2.0:1789760757.964611:0:5431:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51880 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:02bc4b43-5e6f-4d7e-9224-80f5c789104f:5431:1876700024543232:192.168.203.107@tcp:503:ll_cfg_requeue.0 00000100:00100000:2.0:1789760757.968095:0:5965:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51880 pname:cluuid:pid:xid:nid:opc:job llog_process_th:02bc4b43-5e6f-4d7e-9224-80f5c789104f:5965:1876700024543360:192.168.203.107@tcp:502:llog_process_th.0 00000100:00100000:2.0:1789760757.968239:0:5965:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c056baf00 going to sleep for 11 seconds 00000100:00100000:2.0:1789760757.971132:0:5965:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51880 pname:cluuid:pid:xid:nid:opc:job llog_process_th:02bc4b43-5e6f-4d7e-9224-80f5c789104f:5965:1876700024543360:192.168.203.107@tcp:502:llog_process_th.0 00000020:01000000:2.0:1789760757.971199:0:5965:0:(obd_config.c:1893:class_config_llog_handler()) Marker, inst_flg=0x0 mark_flg=0x1 00000020:00000080:2.0:1789760757.971213:0:5965:0:(obd_config.c:1472:class_process_config()) processing cmd: cf010 00000020:00000080:2.0:1789760757.971215:0:5965:0:(obd_config.c:1563:class_process_config()) marker 3 (0x1) general lod.*.mdt_hash 00000020:01000000:2.0:1789760757.971224:0:5965:0:(obd_config.c:2004:class_config_llog_handler()) cmd ce032, instance name: general-ffffa06c18a1e000 00000020:00000080:2.0:1789760757.971227:0:5965:0:(obd_config.c:1472:class_process_config()) processing cmd: ce032 00000100:00100000:2.0:1789760757.971476:0:5965:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51880 pname:cluuid:pid:xid:nid:opc:job llog_process_th:02bc4b43-5e6f-4d7e-9224-80f5c789104f:5965:1876700024543488:192.168.203.107@tcp:502:llog_process_th.0 00000100:00100000:2.0:1789760757.971597:0:5965:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c056baf00 going to sleep for 11 seconds 00000100:00100000:0.0:1789760757.986330:0:5965:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51880 pname:cluuid:pid:xid:nid:opc:job llog_process_th:02bc4b43-5e6f-4d7e-9224-80f5c789104f:5965:1876700024543488:192.168.203.107@tcp:502:llog_process_th.0 00000020:01000000:0.0:1789760757.986404:0:5965:0:(obd_config.c:1893:class_config_llog_handler()) Marker, inst_flg=0x2 mark_flg=0x2 00000020:00000080:0.0:1789760757.986418:0:5965:0:(obd_config.c:1472:class_process_config()) processing cmd: cf010 00000020:00000080:0.0:1789760757.986421:0:5965:0:(obd_config.c:1563:class_process_config()) marker 3 (0x2) general lod.*.mdt_hash 00000040:00080000:0.0:1789760757.986427:0:5965:0:(llog.c:862:llog_process_thread()) stop processing plain [0xa:0x2:0x0] index 7 count 7 00000020:01000000:1.0:1789760757.990180:0:5431:0:(obd_config.c:2145:class_config_parse_llog()) Processed log params gen 4-6 (rc=0) 10000000:01000000:1.0:1789760757.990224:0:5431:0:(mgc_request.c:1879:mgc_process_log()) MGC192.168.203.107@tcp: configuration from log 'params' succeeded (0). 00010000:00010000:1.0:1789760757.990239:0:5431:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.203.107@tcp lock: ffffa06c01512e00/0x59bc99bb619480cf lrc: 3/1,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x1daf3002fe9e0939 expref: -99 pid: 5431 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1789760757.990254:0:5431:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: MGC192.168.203.107@tcp lock: ffffa06c01512e00/0x59bc99bb619480cf lrc: 2/0,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x1daf3002fe9e0939 expref: -99 pid: 5431 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1789760758.394657:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789760759.418530:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789760760.439065:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789760761.466064:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000001:02000400:3.0:1789760761.639927:0:6263:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: oleg307-client.virtnet: executing check_logdir /tmp/testlogs/ 00000100:00100000:1.0:1789760761.720609:0:2342:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2342:1876700024543616:192.168.203.107@tcp:400:kworker.0 00000100:00100000:1.0:1789760761.720712:0:2342:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2342:1876700024543872:192.168.203.107@tcp:400:kworker.0 00000100:00100000:1.0:1789760761.720745:0:2342:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [0->1] 00000100:00100000:1.0:1789760761.720752:0:2342:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2342:1876700024543744:192.168.203.107@tcp:400:kworker.0 00000100:00100000:1.0:1789760761.720777:0:2342:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2342:1876700024544000:192.168.203.107@tcp:400:kworker.0 00010000:00080000:1.0:1789760761.727649:0:2342:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61549) req@ffffa06c08e2e300 x1876700024543872/t0(0) o400->lustre-OST0000-osc-ffffa06c18a1e000@192.168.203.107@tcp:28/4 lens 224/224 e 0 to 0 dl 1789760777 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789760761.727668:0:2342:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2342:1876700024543872:192.168.203.107@tcp:400:kworker.0 00000100:00100000:1.0:1789760761.727698:0:2342:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2342:1876700024543744:192.168.203.107@tcp:400:kworker.0 00010000:00080000:1.0:1789760761.727711:0:2342:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61549) req@ffffa06c08e2c000 x1876700024544000/t0(0) o400->lustre-OST0001-osc-ffffa06c18a1e000@192.168.203.107@tcp:28/4 lens 224/224 e 0 to 0 dl 1789760777 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789760761.727719:0:2342:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2342:1876700024544000:192.168.203.107@tcp:400:kworker.0 00000100:00100000:1.0:1789760761.727729:0:2342:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2342:1876700024543616:192.168.203.107@tcp:400:kworker.0 00010000:00010000:2.0:1789760762.491031:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789760763.517971:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789760764.535092:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000001:02000400:2.0:1789760765.487199:0:6542:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: oleg307-client.virtnet: executing yml_node 00010000:00010000:2.0:1789760765.559114:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789760766.585901:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789760766.840267:0:2342:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2342:1876700024544128:192.168.203.107@tcp:400:kworker.0 00000100:00100000:1.0:1789760766.840385:0:2342:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2342:1876700024544384:192.168.203.107@tcp:400:kworker.0 00000100:00100000:1.0:1789760766.840423:0:2342:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [0->1] 00000100:00100000:1.0:1789760766.840430:0:2342:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2342:1876700024544256:192.168.203.107@tcp:400:kworker.0 00000100:00100000:1.0:1789760766.840460:0:2342:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2342:1876700024544512:192.168.203.107@tcp:400:kworker.0 00000100:00100000:1.0:1789760766.852130:0:2342:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2342:1876700024544256:192.168.203.107@tcp:400:kworker.0 00010000:00080000:1.0:1789760766.852155:0:2342:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61549) req@ffffa06c08e2e300 x1876700024544512/t0(0) o400->lustre-OST0001-osc-ffffa06c18a1e000@192.168.203.107@tcp:28/4 lens 224/224 e 0 to 0 dl 1789760782 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789760766.852169:0:2342:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2342:1876700024544512:192.168.203.107@tcp:400:kworker.0 00000100:00100000:1.0:1789760766.852180:0:2342:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2342:1876700024544128:192.168.203.107@tcp:400:kworker.0 00010000:00080000:1.0:1789760766.852192:0:2342:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61549) req@ffffa06c08e2d180 x1876700024544384/t0(0) o400->lustre-OST0000-osc-ffffa06c18a1e000@192.168.203.107@tcp:28/4 lens 224/224 e 0 to 0 dl 1789760782 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789760766.852200:0:2342:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2342:1876700024544384:192.168.203.107@tcp:400:kworker.0 00010000:00010000:3.0:1789760767.607093:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789760768.632046:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000001:02000400:3.0:1789760769.407294:0:6982:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: Client: 2.17.57.103 00010000:00010000:3.0:1789760769.655318:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789760770.678956:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000001:02000400:2.0:1789760771.281601:0:7136:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: MDS: 2.17.57.103 00010000:00010000:3.0:1789760771.703022:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789760771.959494:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c107c9800 lustre-OST0000_UUID: changing import state from FULL to CONNECTING 00000100:02000000:3.0:1789760771.959501:0:36:0:(import.c:2101:ptlrpc_disconnect_and_idle_import()) lustre-OST0000-osc-ffffa06c18a1e000: disconnect after 23s idle 00000100:00100000:2.0:1789760771.961108:0:2343:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2343:1876700024544640:192.168.203.107@tcp:400:kworker.0 00000100:00100000:2.0:1789760771.961236:0:2343:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1789760771.961244:0:2343:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2343:1876700024544768:192.168.203.107@tcp:400:kworker.0 00000100:00080000:3.0:1789760771.964924:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c0918f000 lustre-OST0001_UUID: changing import state from FULL to CONNECTING 00000100:02000000:3.0:1789760771.964931:0:36:0:(import.c:2101:ptlrpc_disconnect_and_idle_import()) lustre-OST0001-osc-ffffa06c18a1e000: disconnect after 23s idle 00000100:00100000:1.0:1789760771.964933:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024544896:192.168.203.107@tcp:9:kworker.0 00000100:00100000:1.0:1789760771.965034:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024545024:192.168.203.107@tcp:9:kworker.0 00000100:00100000:2.0:1789760771.968359:0:2343:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2343:1876700024544768:192.168.203.107@tcp:400:kworker.0 00000100:00100000:2.0:1789760771.968375:0:2343:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2343:1876700024544640:192.168.203.107@tcp:400:kworker.0 00010000:00080000:1.0:1789760771.972059:0:2340:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61549) req@ffffa06c1872ed80 x1876700024545024/t0(0) o9->lustre-OST0001-osc-ffffa06c18a1e000@192.168.203.107@tcp:28/4 lens 224/224 e 0 to 0 dl 1789760781 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789760771.972082:0:2340:0:(import.c:2011:ptlrpc_disconnect_idle_interpret()) @@@ inflight=1, refcount=3: rc = 0 req@ffffa06c1872ed80 x1876700024545024/t0(0) o9->lustre-OST0001-osc-ffffa06c18a1e000@192.168.203.107@tcp:28/4 lens 224/224 e 0 to 0 dl 1789760781 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789760771.972091:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c0918f000 lustre-OST0001_UUID: changing import state from CONNECTING to IDLE 00000100:00100000:1.0:1789760771.972096:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024545024:192.168.203.107@tcp:9:kworker.0 00010000:00080000:1.0:1789760771.972111:0:2340:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61549) req@ffffa06c1872f100 x1876700024544896/t0(0) o9->lustre-OST0000-osc-ffffa06c18a1e000@192.168.203.107@tcp:28/4 lens 224/224 e 0 to 0 dl 1789760781 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789760771.972121:0:2340:0:(import.c:2011:ptlrpc_disconnect_idle_interpret()) @@@ inflight=1, refcount=3: rc = 0 req@ffffa06c1872f100 x1876700024544896/t0(0) o9->lustre-OST0000-osc-ffffa06c18a1e000@192.168.203.107@tcp:28/4 lens 224/224 e 0 to 0 dl 1789760781 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789760771.972141:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c107c9800 lustre-OST0000_UUID: changing import state from CONNECTING to IDLE 00000100:00100000:1.0:1789760771.972146:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024544896:192.168.203.107@tcp:9:kworker.0 00010000:00010000:3.0:1789760772.727001:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000001:02000400:1.0:1789760773.356717:0:7291:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: OSS: 2.17.57.103 00010000:00010000:3.0:1789760773.751803:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000001:02000400:2.0:1789760774.458649:0:7436:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: -----============= acceptance-small: sanityn ============----- Fri Sep 18 15:46:13 EDT 2026 00010000:00010000:3.0:1789760774.775606:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789760775.799030:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789760776.829392:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789760777.079768:0:2343:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2343:1876700024545152:192.168.203.107@tcp:400:kworker.0 00000100:00100000:2.0:1789760777.079902:0:2343:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1789760777.079911:0:2343:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2343:1876700024545280:192.168.203.107@tcp:400:kworker.0 00000100:00100000:2.0:1789760777.083927:0:2343:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2343:1876700024545152:192.168.203.107@tcp:400:kworker.0 00000100:00100000:2.0:1789760777.084126:0:2343:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2343:1876700024545280:192.168.203.107@tcp:400:kworker.0 00010000:00010000:3.0:1789760777.847043:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789760778.871170:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789760779.896629:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789760780.932323:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789760781.943469:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789760782.198956:0:2343:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2343:1876700024545408:192.168.203.107@tcp:400:kworker.0 00000100:00100000:3.0:1789760782.199059:0:2343:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1789760782.199066:0:2343:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2343:1876700024545536:192.168.203.107@tcp:400:kworker.0 00000100:00100000:3.0:1789760782.202689:0:2343:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2343:1876700024545536:192.168.203.107@tcp:400:kworker.0 00000100:00100000:3.0:1789760782.202706:0:2343:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2343:1876700024545408:192.168.203.107@tcp:400:kworker.0 00010000:00010000:0.0:1789760782.967519:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789760783.993358:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789760785.019758:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789760786.039183:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000001:02000400:1.0:1789760786.856304:0:7952:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: excepting tests: 40a 00010000:00010000:0.0:1789760787.063811:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789760787.319745:0:2342:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2342:1876700024545664:192.168.203.107@tcp:400:kworker.0 00000100:00100000:1.0:1789760787.319881:0:2342:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1789760787.319888:0:2342:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2342:1876700024545792:192.168.203.107@tcp:400:kworker.0 00000100:00100000:1.0:1789760787.323420:0:2342:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2342:1876700024545792:192.168.203.107@tcp:400:kworker.0 00000100:00100000:1.0:1789760787.323437:0:2342:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2342:1876700024545664:192.168.203.107@tcp:400:kworker.0 00000001:02000400:1.0:1789760788.022932:0:8089:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: skipping tests SLOW=no: 33a 00010000:00010000:1.0:1789760788.089601:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789760789.111368:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000001:02000400:2.0:1789760789.447397:0:8232:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: === sanityn: start setup 15:46:28 (1789760788) === 00000080:00200008:1.0:1789760789.520460:0:7453:0:(file.c:6296:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffffa06c03518908), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1789760789.520474:0:7453:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908),name=/ 00000002:00010000:1.0:1789760789.520497:0:7453:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1789760789.520517:0:7453:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c19d4c000/0x59bc99bb619480ba lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1daf3002fe9e0898 expref: -99 pid: 5456 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 1 00010000:00010000:1.0:1789760789.520533:0:7453:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c19d4c000/0x59bc99bb619480ba lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1daf3002fe9e0898 expref: -99 pid: 5456 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 1 00000080:00200000:1.0:1789760789.520541:0:7453:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1789760789.520550:0:7453:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa06c03518908) 00000080:00010000:1.0:1789760789.520559:0:7453:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa06c03518908) for lock 0x59bc99bb619480ba 00000080:00010000:1.0:1789760789.520567:0:7453:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x59bc99bb619480ba from it ffffafafc1c3bcb8 00010000:00010000:1.0:1789760789.520572:0:7453:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c19d4c000/0x59bc99bb619480ba lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1daf3002fe9e0898 expref: -99 pid: 5456 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 1 00010000:00010000:1.0:1789760789.520581:0:7453:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c19d4c000/0x59bc99bb619480ba lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1daf3002fe9e0898 expref: -99 pid: 5456 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 1 00000080:00200008:3.0:1789760789.548757:0:7453:0:(file.c:6296:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffffa06c03518908), request_mask 2047, flags 0, foreign 0 00000080:00200000:3.0:1789760789.548788:0:7453:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908),name=/ 00000002:00010000:3.0:1789760789.548810:0:7453:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1789760789.548828:0:7453:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c19d4c000/0x59bc99bb619480ba lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1daf3002fe9e0898 expref: -99 pid: 5456 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 1 00010000:00010000:3.0:1789760789.548864:0:7453:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c19d4c000/0x59bc99bb619480ba lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1daf3002fe9e0898 expref: -99 pid: 5456 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 1 00000080:00200000:3.0:1789760789.548874:0:7453:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1789760789.548881:0:7453:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa06c03518908) 00000080:00010000:3.0:1789760789.548885:0:7453:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa06c03518908) for lock 0x59bc99bb619480ba 00000080:00010000:3.0:1789760789.548893:0:7453:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x59bc99bb619480ba from it ffffafafc1c3bcb8 00010000:00010000:3.0:1789760789.548897:0:7453:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c19d4c000/0x59bc99bb619480ba lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1daf3002fe9e0898 expref: -99 pid: 5456 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 1 00010000:00010000:3.0:1789760789.548919:0:7453:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c19d4c000/0x59bc99bb619480ba lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1daf3002fe9e0898 expref: -99 pid: 5456 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 1 00000080:00200008:1.0:1789760789.575148:0:7453:0:(file.c:6296:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffffa06c03518908), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1789760789.575161:0:7453:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908),name=/ 00000002:00010000:1.0:1789760789.575182:0:7453:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1789760789.575200:0:7453:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c19d4c000/0x59bc99bb619480ba lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1daf3002fe9e0898 expref: -99 pid: 5456 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 1 00010000:00010000:1.0:1789760789.575218:0:7453:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c19d4c000/0x59bc99bb619480ba lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1daf3002fe9e0898 expref: -99 pid: 5456 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 1 00000080:00200000:1.0:1789760789.575227:0:7453:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1789760789.575234:0:7453:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa06c03518908) 00000080:00010000:1.0:1789760789.575237:0:7453:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa06c03518908) for lock 0x59bc99bb619480ba 00000080:00010000:1.0:1789760789.575245:0:7453:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x59bc99bb619480ba from it ffffafafc1c3bcb8 00010000:00010000:1.0:1789760789.575248:0:7453:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c19d4c000/0x59bc99bb619480ba lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1daf3002fe9e0898 expref: -99 pid: 5456 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 1 00010000:00010000:1.0:1789760789.575256:0:7453:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c19d4c000/0x59bc99bb619480ba lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1daf3002fe9e0898 expref: -99 pid: 5456 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 1 00000080:00200008:3.0:1789760789.589162:0:7453:0:(file.c:6296:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffffa06c03518908), request_mask 2047, flags 0, foreign 0 00000080:00200000:3.0:1789760789.589173:0:7453:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908),name=/ 00000002:00010000:3.0:1789760789.589193:0:7453:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1789760789.589210:0:7453:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c19d4c000/0x59bc99bb619480ba lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1daf3002fe9e0898 expref: -99 pid: 5456 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 1 00010000:00010000:3.0:1789760789.589224:0:7453:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c19d4c000/0x59bc99bb619480ba lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1daf3002fe9e0898 expref: -99 pid: 5456 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 1 00000080:00200000:3.0:1789760789.589232:0:7453:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1789760789.589239:0:7453:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa06c03518908) 00000080:00010000:3.0:1789760789.589242:0:7453:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa06c03518908) for lock 0x59bc99bb619480ba 00000080:00010000:3.0:1789760789.589248:0:7453:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x59bc99bb619480ba from it ffffafafc1c3bcb8 00010000:00010000:3.0:1789760789.589252:0:7453:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c19d4c000/0x59bc99bb619480ba lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1daf3002fe9e0898 expref: -99 pid: 5456 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 1 00010000:00010000:3.0:1789760789.589260:0:7453:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c19d4c000/0x59bc99bb619480ba lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1daf3002fe9e0898 expref: -99 pid: 5456 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 1 00000020:01000004:2.0:1789760789.844709:0:8280:0:(obd_mount.c:923:lmd_print()) mount data: 00000020:01000004:2.0:1789760789.844713:0:8280:0:(obd_mount.c:925:lmd_print()) profile: lustre-client 00000020:01000004:2.0:1789760789.844715:0:8280:0:(obd_mount.c:926:lmd_print()) device: 192.168.203.107@tcp:/lustre 00000020:01000004:2.0:1789760789.844716:0:8280:0:(obd_mount.c:929:lmd_print()) options: user_xattr,flock 00000080:01200004:2.0:1789760789.845348:0:8280:0:(super25.c:131:lustre_fill_super()) VFS Op: sb ffffa06c07e46000 00000080:01000004:2.0:1789760789.845357:0:8280:0:(super25.c:162:lustre_fill_super()) Mounting client lustre-client 00000020:01000004:2.0:1789760789.845376:0:8280:0:(obd_mount.c:403:lustre_start_mgc()) MGC192.168.203.107@tcp:Set MGC reconnect 1 10000000:01000000:2.0:1789760789.845379:0:8280:0:(mgc_request.c:1071:mgc_set_info_async()) InitRecov MGC192.168.203.107@tcp 1/d0:i0:r0:or0:FULL 00000080:00200000:2.0:1789760789.845384:0:8280:0:(llite_lib.c:1506:ll_fill_super()) VFS Op: cfg_instance lustre-client-ffffa06c07e46000 (sb ffffa06c07e46000) 00000080:01000000:2.0:1789760789.845452:0:8280:0:(llite_lib.c:1232:ll_options()) Parsing opts user_xattr,flock 00000080:00000004:2.0:1789760789.845453:0:8280:0:(llite_lib.c:1242:ll_options()) next opt=user_xattr 00000080:00000004:2.0:1789760789.845456:0:8280:0:(llite_lib.c:1242:ll_options()) next opt=flock 00000080:01000000:2.0:1789760789.845462:0:8280:0:(llite_lib.c:1540:ll_fill_super()) llite sb uuid: b837c2fb-d1c1-4622-807e-a243f6fc2450 10000000:01000000:3.0:1789760789.849152:0:8280:0:(mgc_request.c:1944:mgc_process_config()) parse_log lustre-client from 0 10000000:01000000:3.0:1789760789.849158:0:8280:0:(mgc_request.c:329:config_log_add()) add config log lustre-client-ffffa06c07e46000 10000000:01000000:3.0:1789760789.849165:0:8280:0:(mgc_request.c:194:do_config_log_add()) do adding config log params-ffffa06c07e46000 10000000:01000000:3.0:1789760789.849169:0:8280:0:(mgc_request.c:68:mgc_name2resid()) log params to resid 0x736d61726170/0x3 (params) 10000000:01000000:3.0:1789760789.849173:0:8280:0:(mgc_request.c:194:do_config_log_add()) do adding config log lustre-client-ffffa06c07e46000 10000000:01000000:3.0:1789760789.849183:0:8280:0:(mgc_request.c:68:mgc_name2resid()) log lustre-client to resid 0x65727473756c/0x0 (lustre) 10000000:01000000:3.0:1789760789.849187:0:8280:0:(mgc_request.c:194:do_config_log_add()) do adding config log lustre-cliir-ffffa06c07e46000 10000000:01000000:3.0:1789760789.849189:0:8280:0:(mgc_request.c:68:mgc_name2resid()) log lustre-cliir to resid 0x65727473756c/0x2 (lustre) 10000000:01000000:3.0:1789760789.849193:0:8280:0:(mgc_request.c:1802:mgc_process_log()) Process log lustre-client-ffffa06c07e46000 from 1 10000000:01000000:3.0:1789760789.849195:0:8280:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-client (res 0x65727473756c) 00010000:00010000:3.0:1789760789.849233:0:8280:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.203.107@tcp lock: ffffa06c42174a00/0x59bc99bb619480d6 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 4 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 8280 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1789760789.849246:0:8280:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.203.107@tcp lock: ffffa06c42174a00/0x59bc99bb619480d6 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 4 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 8280 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1789760789.849257:0:8280:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.203.107@tcp lock: ffffa06c42174a00/0x59bc99bb619480d6 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 4 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 8280 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1789760789.849281:0:8280:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872e300 pname:cluuid:pid:xid:nid:opc:job mount.lustre:02bc4b43-5e6f-4d7e-9224-80f5c789104f:8280:1876700024545920:192.168.203.107@tcp:101:mount.lustre.0 00000100:00100000:3.0:1789760789.855139:0:8280:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c108c9a80 going to sleep for 11 seconds 00000100:00100000:3.0:1789760789.858210:0:8280:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872e300 pname:cluuid:pid:xid:nid:opc:job mount.lustre:02bc4b43-5e6f-4d7e-9224-80f5c789104f:8280:1876700024545920:192.168.203.107@tcp:101:mount.lustre.0 00010000:00010000:3.0:1789760789.858238:0:8280:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: MGC192.168.203.107@tcp lock: ffffa06c42174a00/0x59bc99bb619480d6 lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 4 type: PLN flags: 0x11000000000000 nid: local remote: 0x1daf3002fe9e094e expref: -99 pid: 8280 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1789760789.858250:0:8280:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: MGC192.168.203.107@tcp lock: ffffa06c42174a00/0x59bc99bb619480d6 lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 4 type: PLN flags: 0x1000000000000 nid: local remote: 0x1daf3002fe9e094e expref: -99 pid: 8280 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1789760789.858321:0:8280:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872e300 pname:cluuid:pid:xid:nid:opc:job mount.lustre:02bc4b43-5e6f-4d7e-9224-80f5c789104f:8280:1876700024546048:192.168.203.107@tcp:501:mount.lustre.0 00000100:00100000:3.0:1789760789.858425:0:8280:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c108c9a80 going to sleep for 11 seconds 00000100:00100000:3.0:1789760789.862112:0:8280:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872e300 pname:cluuid:pid:xid:nid:opc:job mount.lustre:02bc4b43-5e6f-4d7e-9224-80f5c789104f:8280:1876700024546048:192.168.203.107@tcp:501:mount.lustre.0 00000100:00100000:3.0:1789760789.862173:0:8280:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872e300 pname:cluuid:pid:xid:nid:opc:job mount.lustre:02bc4b43-5e6f-4d7e-9224-80f5c789104f:8280:1876700024546176:192.168.203.107@tcp:503:mount.lustre.0 00000100:00100000:3.0:1789760789.862263:0:8280:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c108c9a80 going to sleep for 11 seconds 00000100:00100000:3.0:1789760789.864112:0:8280:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872e300 pname:cluuid:pid:xid:nid:opc:job mount.lustre:02bc4b43-5e6f-4d7e-9224-80f5c789104f:8280:1876700024546176:192.168.203.107@tcp:503:mount.lustre.0 00000100:00100000:3.0:1789760789.864514:0:8284:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872e300 pname:cluuid:pid:xid:nid:opc:job llog_process_th:02bc4b43-5e6f-4d7e-9224-80f5c789104f:8284:1876700024546304:192.168.203.107@tcp:502:llog_process_th.0 00000100:00100000:3.0:1789760789.864590:0:8284:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c108c96c0 going to sleep for 11 seconds 00000100:00100000:3.0:1789760789.867265:0:8284:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872e300 pname:cluuid:pid:xid:nid:opc:job llog_process_th:02bc4b43-5e6f-4d7e-9224-80f5c789104f:8284:1876700024546304:192.168.203.107@tcp:502:llog_process_th.0 00000020:01000000:3.0:1789760789.867304:0:8284:0:(obd_config.c:1893:class_config_llog_handler()) Marker, inst_flg=0x0 mark_flg=0x1 00000020:00000080:3.0:1789760789.867314:0:8284:0:(obd_config.c:1472:class_process_config()) processing cmd: cf010 00000020:00000080:3.0:1789760789.867317:0:8284:0:(obd_config.c:1563:class_process_config()) marker 4 (0x1) lustre-clilov lov setup 00000020:01000000:3.0:1789760789.867324:0:8284:0:(obd_config.c:2004:class_config_llog_handler()) cmd cf001, instance name: lustre-clilov-ffffa06c07e46000 00000020:00000080:3.0:1789760789.867327:0:8284:0:(obd_config.c:1472:class_process_config()) processing cmd: cf001 00000020:00000080:3.0:1789760789.867349:0:8284:0:(genops.c:413:class_newdev()) Allocate new device lustre-clilov-ffffa06c07e46000 (ffffa06c181e0000) 00000020:00000080:3.0:1789760789.867375:0:8284:0:(obd_config.c:720:class_attach_name()) OBD: dev 6 attached type lov with refcount 2 00000020:01000000:3.0:1789760789.867383:0:8284:0:(obd_config.c:2004:class_config_llog_handler()) cmd cf003, instance name: lustre-clilov-ffffa06c07e46000 00000020:00000080:3.0:1789760789.867386:0:8284:0:(obd_config.c:1472:class_process_config()) processing cmd: cf003 00000020:00000080:3.0:1789760789.867663:0:8284:0:(obd_config.c:826:class_setup()) finished setup of obd lustre-clilov-ffffa06c07e46000 (uuid b837c2fb-d1c1-4622-807e-a243f6fc2450) 00000020:01000000:3.0:1789760789.867668:0:8284:0:(obd_config.c:1893:class_config_llog_handler()) Marker, inst_flg=0x2 mark_flg=0x2 00000020:00000080:3.0:1789760789.867672:0:8284:0:(obd_config.c:1472:class_process_config()) processing cmd: cf010 00000020:00000080:3.0:1789760789.867674:0:8284:0:(obd_config.c:1563:class_process_config()) marker 4 (0x2) lustre-clilov lov setup 00000020:01000000:3.0:1789760789.867677:0:8284:0:(obd_config.c:1893:class_config_llog_handler()) Marker, inst_flg=0x0 mark_flg=0x1 00000020:00000080:3.0:1789760789.867680:0:8284:0:(obd_config.c:1472:class_process_config()) processing cmd: cf010 00000020:00000080:3.0:1789760789.867681:0:8284:0:(obd_config.c:1563:class_process_config()) marker 5 (0x1) lustre-clilmv lmv setup 00000020:01000000:3.0:1789760789.867685:0:8284:0:(obd_config.c:2004:class_config_llog_handler()) cmd cf001, instance name: lustre-clilmv-ffffa06c07e46000 00000020:00000080:3.0:1789760789.867687:0:8284:0:(obd_config.c:1472:class_process_config()) processing cmd: cf001 00000020:00000080:3.0:1789760789.867695:0:8284:0:(genops.c:413:class_newdev()) Allocate new device lustre-clilmv-ffffa06c07e46000 (ffffa06c181e3370) 00000020:00000080:3.0:1789760789.867706:0:8284:0:(obd_config.c:720:class_attach_name()) OBD: dev 7 attached type lmv with refcount 2 00000020:01000000:3.0:1789760789.867710:0:8284:0:(obd_config.c:2004:class_config_llog_handler()) cmd cf003, instance name: lustre-clilmv-ffffa06c07e46000 00000020:00000080:3.0:1789760789.867712:0:8284:0:(obd_config.c:1472:class_process_config()) processing cmd: cf003 00000020:00000080:3.0:1789760789.867909:0:8284:0:(obd_config.c:826:class_setup()) finished setup of obd lustre-clilmv-ffffa06c07e46000 (uuid b837c2fb-d1c1-4622-807e-a243f6fc2450) 00000020:01000000:3.0:1789760789.867920:0:8284:0:(obd_config.c:1893:class_config_llog_handler()) Marker, inst_flg=0x2 mark_flg=0x2 00000020:00000080:3.0:1789760789.867923:0:8284:0:(obd_config.c:1472:class_process_config()) processing cmd: cf010 00000020:00000080:3.0:1789760789.867925:0:8284:0:(obd_config.c:1563:class_process_config()) marker 5 (0x2) lustre-clilmv lmv setup 00000020:01000000:3.0:1789760789.867927:0:8284:0:(obd_config.c:1893:class_config_llog_handler()) Marker, inst_flg=0x0 mark_flg=0x1 00000020:00000080:3.0:1789760789.867930:0:8284:0:(obd_config.c:1472:class_process_config()) processing cmd: cf010 00000020:00000080:3.0:1789760789.867931:0:8284:0:(obd_config.c:1563:class_process_config()) marker 6 (0x1) lustre-client mount opts 00000020:00000080:3.0:1789760789.867934:0:8284:0:(obd_config.c:1472:class_process_config()) processing cmd: cf007 00000020:00000080:3.0:1789760789.867936:0:8284:0:(obd_config.c:1516:class_process_config()) mountopt: profile lustre-client osc lustre-clilov mdc lustre-clilmv 00000020:01000000:3.0:1789760789.867938:0:8284:0:(obd_config.c:1149:class_add_profile()) Add profile lustre-client 00000020:01000000:3.0:1789760789.867943:0:8284:0:(obd_config.c:1893:class_config_llog_handler()) Marker, inst_flg=0x2 mark_flg=0x2 00000020:00000080:3.0:1789760789.867945:0:8284:0:(obd_config.c:1472:class_process_config()) processing cmd: cf010 00000020:00000080:3.0:1789760789.867947:0:8284:0:(obd_config.c:1563:class_process_config()) marker 6 (0x2) lustre-client mount opts 00000020:01000000:3.0:1789760789.867949:0:8284:0:(obd_config.c:1893:class_config_llog_handler()) Marker, inst_flg=0x0 mark_flg=0x1 00000020:00000080:3.0:1789760789.867952:0:8284:0:(obd_config.c:1472:class_process_config()) processing cmd: cf010 00000020:00000080:3.0:1789760789.867954:0:8284:0:(obd_config.c:1563:class_process_config()) marker 7 (0x1) lustre-MDT0000 add mdc 00000020:00000080:3.0:1789760789.867956:0:8284:0:(obd_config.c:1472:class_process_config()) processing cmd: cf005 00000020:00000080:3.0:1789760789.867960:0:8284:0:(obd_config.c:1487:class_process_config()) adding mapping from uuid 192.168.203.107@tcp to nid 0x20000c0a8cb6b (192.168.203.107@tcp) 00000020:01000000:3.0:1789760789.867980:0:8284:0:(obd_config.c:2004:class_config_llog_handler()) cmd cf001, instance name: lustre-MDT0000-mdc-ffffa06c07e46000 00000020:00000080:3.0:1789760789.867983:0:8284:0:(obd_config.c:1472:class_process_config()) processing cmd: cf001 00000020:00000080:3.0:1789760789.867989:0:8284:0:(genops.c:413:class_newdev()) Allocate new device lustre-MDT0000-mdc-ffffa06c07e46000 (ffffa06c181e4d28) 00000020:00000080:3.0:1789760789.867998:0:8284:0:(obd_config.c:720:class_attach_name()) OBD: dev 8 attached type mdc with refcount 2 00000020:01000000:3.0:1789760789.868003:0:8284:0:(obd_config.c:2004:class_config_llog_handler()) cmd cf003, instance name: lustre-MDT0000-mdc-ffffa06c07e46000 00000020:00000080:3.0:1789760789.868005:0:8284:0:(obd_config.c:1472:class_process_config()) processing cmd: cf003 00010000:00080000:3.0:1789760789.868070:0:8284:0:(ldlm_lib.c:102:import_set_conn()) imp ffffa06c09137000@lustre-MDT0000-mdc-ffffa06c07e46000: add connection 192.168.203.107@tcp at head 00010000:00010000:3.0:1789760789.877095:0:8284:0:(ldlm_pool.c:923:ldlm_pool_init()) Lock pool ldlm-pool-lustre-MDT0000-mdc-ffffa06c07e46000-4 is initialized 00000040:01000000:3.0:1789760789.877354:0:8284:0:(llog_obd.c:197:llog_setup()) obd lustre-MDT0000-mdc-ffffa06c07e46000 ctxt 13 is initialized 00000020:00000080:3.0:1789760789.877399:0:8284:0:(obd_config.c:826:class_setup()) finished setup of obd lustre-MDT0000-mdc-ffffa06c07e46000 (uuid b837c2fb-d1c1-4622-807e-a243f6fc2450) 00000020:01000000:3.0:1789760789.877413:0:8284:0:(obd_config.c:2004:class_config_llog_handler()) cmd cf014, instance name: lustre-clilmv-ffffa06c07e46000 00000020:00000080:3.0:1789760789.877417:0:8284:0:(obd_config.c:1472:class_process_config()) processing cmd: cf014 00800000:01000000:3.0:1789760789.877440:0:8284:0:(lmv_obd.c:442:lmv_add_target()) Target uuid: lustre-MDT0000_UUID. index 0 00000020:01000000:3.0:1789760789.877476:0:8284:0:(obd_config.c:1893:class_config_llog_handler()) Marker, inst_flg=0x2 mark_flg=0x2 00000020:00000080:3.0:1789760789.877485:0:8284:0:(obd_config.c:1472:class_process_config()) processing cmd: cf010 00000020:00000080:3.0:1789760789.877487:0:8284:0:(obd_config.c:1563:class_process_config()) marker 7 (0x2) lustre-MDT0000 add mdc 00000020:01000000:3.0:1789760789.877490:0:8284:0:(obd_config.c:1893:class_config_llog_handler()) Marker, inst_flg=0x0 mark_flg=0x5 00000020:01000000:3.0:1789760789.877492:0:8284:0:(obd_config.c:1902:class_config_llog_handler()) SKIP #9 00000020:01000000:3.0:1789760789.877494:0:8284:0:(obd_config.c:1928:class_config_llog_handler()) skipping 0x6 00000020:01000000:3.0:1789760789.877495:0:8284:0:(obd_config.c:1928:class_config_llog_handler()) skipping 0x6 00000020:01000000:3.0:1789760789.877497:0:8284:0:(obd_config.c:1893:class_config_llog_handler()) Marker, inst_flg=0x6 mark_flg=0x6 00000020:00000080:3.0:1789760789.877499:0:8284:0:(obd_config.c:1472:class_process_config()) processing cmd: cf010 00000020:00000080:3.0:1789760789.877500:0:8284:0:(obd_config.c:1563:class_process_config()) marker 9 (0x6) lustre sys.timeout 00000020:01000000:3.0:1789760789.877502:0:8284:0:(obd_config.c:1893:class_config_llog_handler()) Marker, inst_flg=0x0 mark_flg=0x1 00000020:01000004:3.0:1789760789.877506:0:8284:0:(obd_mount.c:961:lustre_check_exclusion()) Check exclusion lustre-OST0000 (0) in 0 of 192.168.203.107@tcp:/lustre 00000020:00000080:3.0:1789760789.877509:0:8284:0:(obd_config.c:1472:class_process_config()) processing cmd: cf010 00000020:00000080:3.0:1789760789.877510:0:8284:0:(obd_config.c:1563:class_process_config()) marker 13 (0x1) lustre-OST0000 add osc 00000020:00000080:3.0:1789760789.877512:0:8284:0:(obd_config.c:1472:class_process_config()) processing cmd: cf005 00000020:00000080:3.0:1789760789.877517:0:8284:0:(obd_config.c:1487:class_process_config()) adding mapping from uuid 192.168.203.107@tcp to nid 0x20000c0a8cb6b (192.168.203.107@tcp) 00000020:01000000:3.0:1789760789.877543:0:8284:0:(obd_config.c:2004:class_config_llog_handler()) cmd cf001, instance name: lustre-OST0000-osc-ffffa06c07e46000 00000020:00000080:3.0:1789760789.877545:0:8284:0:(obd_config.c:1472:class_process_config()) processing cmd: cf001 00000020:00000080:3.0:1789760789.877555:0:8284:0:(genops.c:413:class_newdev()) Allocate new device lustre-OST0000-osc-ffffa06c07e46000 (ffffa06c181e19b8) 00000020:00000080:3.0:1789760789.877568:0:8284:0:(obd_config.c:720:class_attach_name()) OBD: dev 9 attached type osc with refcount 2 00000020:01000000:3.0:1789760789.877573:0:8284:0:(obd_config.c:2004:class_config_llog_handler()) cmd cf003, instance name: lustre-OST0000-osc-ffffa06c07e46000 00000020:00000080:3.0:1789760789.877575:0:8284:0:(obd_config.c:1472:class_process_config()) processing cmd: cf003 00010000:00080000:3.0:1789760789.877648:0:8284:0:(ldlm_lib.c:102:import_set_conn()) imp ffffa06c09133000@lustre-OST0000-osc-ffffa06c07e46000: add connection 192.168.203.107@tcp at head 00010000:00010000:3.0:1789760789.878374:0:8284:0:(ldlm_pool.c:923:ldlm_pool_init()) Lock pool ldlm-pool-lustre-OST0000-osc-ffffa06c07e46000-5 is initialized 00000020:00000080:3.0:1789760789.878563:0:8284:0:(obd_config.c:826:class_setup()) finished setup of obd lustre-OST0000-osc-ffffa06c07e46000 (uuid b837c2fb-d1c1-4622-807e-a243f6fc2450) 00000020:01000000:3.0:1789760789.878573:0:8284:0:(obd_config.c:2004:class_config_llog_handler()) cmd cf00d, instance name: lustre-clilov-ffffa06c07e46000 00000020:00000080:3.0:1789760789.878575:0:8284:0:(obd_config.c:1472:class_process_config()) processing cmd: cf00d 00020000:01000000:3.0:1789760789.878590:0:8284:0:(lov_obd.c:481:lov_add_target()) uuid:lustre-OST0000_UUID idx:0 gen:1 active:1 00020000:01000000:3.0:1789760789.878596:0:8284:0:(lov_obd.c:510:lov_add_target()) lustre-clilov-ffffa06c07e46000: Adding target lustre-OST0000_UUID 00020000:01000000:3.0:1789760789.878600:0:8284:0:(lov_obd.c:532:lov_add_target()) idx=0 ltd_gen=1 ld_tgt_count=1 00000020:01000000:3.0:1789760789.878611:0:8284:0:(obd_config.c:1893:class_config_llog_handler()) Marker, inst_flg=0x2 mark_flg=0x2 00000020:00000080:3.0:1789760789.878614:0:8284:0:(obd_config.c:1472:class_process_config()) processing cmd: cf010 00000020:00000080:3.0:1789760789.878616:0:8284:0:(obd_config.c:1563:class_process_config()) marker 13 (0x2) lustre-OST0000 add osc 00000020:01000000:3.0:1789760789.878624:0:8284:0:(obd_config.c:1893:class_config_llog_handler()) Marker, inst_flg=0x0 mark_flg=0x5 00000020:01000000:3.0:1789760789.878626:0:8284:0:(obd_config.c:1902:class_config_llog_handler()) SKIP #16 00000020:01000000:3.0:1789760789.878628:0:8284:0:(obd_config.c:1928:class_config_llog_handler()) skipping 0x6 00000020:01000000:3.0:1789760789.878629:0:8284:0:(obd_config.c:1928:class_config_llog_handler()) skipping 0x6 00000020:01000000:3.0:1789760789.878631:0:8284:0:(obd_config.c:1893:class_config_llog_handler()) Marker, inst_flg=0x6 mark_flg=0x6 00000020:00000080:3.0:1789760789.878633:0:8284:0:(obd_config.c:1472:class_process_config()) processing cmd: cf010 00000020:00000080:3.0:1789760789.878634:0:8284:0:(obd_config.c:1563:class_process_config()) marker 16 (0x6) lustre sys.timeout 00000020:01000000:3.0:1789760789.878636:0:8284:0:(obd_config.c:1893:class_config_llog_handler()) Marker, inst_flg=0x0 mark_flg=0x1 00000020:01000004:3.0:1789760789.878639:0:8284:0:(obd_mount.c:961:lustre_check_exclusion()) Check exclusion lustre-OST0001 (1) in 0 of 192.168.203.107@tcp:/lustre 00000020:00000080:3.0:1789760789.878642:0:8284:0:(obd_config.c:1472:class_process_config()) processing cmd: cf010 00000020:00000080:3.0:1789760789.878643:0:8284:0:(obd_config.c:1563:class_process_config()) marker 19 (0x1) lustre-OST0001 add osc 00000020:00000080:3.0:1789760789.878645:0:8284:0:(obd_config.c:1472:class_process_config()) processing cmd: cf005 00000020:00000080:3.0:1789760789.878648:0:8284:0:(obd_config.c:1487:class_process_config()) adding mapping from uuid 192.168.203.107@tcp to nid 0x20000c0a8cb6b (192.168.203.107@tcp) 00000020:01000000:3.0:1789760789.878662:0:8284:0:(obd_config.c:2004:class_config_llog_handler()) cmd cf001, instance name: lustre-OST0001-osc-ffffa06c07e46000 00000020:00000080:3.0:1789760789.878664:0:8284:0:(obd_config.c:1472:class_process_config()) processing cmd: cf001 00000020:00000080:3.0:1789760789.878677:0:8284:0:(genops.c:413:class_newdev()) Allocate new device lustre-OST0001-osc-ffffa06c07e46000 (ffffa06c10dd99b8) 00000020:00000080:3.0:1789760789.878688:0:8284:0:(obd_config.c:720:class_attach_name()) OBD: dev 10 attached type osc with refcount 2 00000020:01000000:3.0:1789760789.878694:0:8284:0:(obd_config.c:2004:class_config_llog_handler()) cmd cf003, instance name: lustre-OST0001-osc-ffffa06c07e46000 00000020:00000080:3.0:1789760789.878696:0:8284:0:(obd_config.c:1472:class_process_config()) processing cmd: cf003 00010000:00080000:3.0:1789760789.878746:0:8284:0:(ldlm_lib.c:102:import_set_conn()) imp ffffa06c09134000@lustre-OST0001-osc-ffffa06c07e46000: add connection 192.168.203.107@tcp at head 00010000:00010000:3.0:1789760789.878977:0:8284:0:(ldlm_pool.c:923:ldlm_pool_init()) Lock pool ldlm-pool-lustre-OST0001-osc-ffffa06c07e46000-6 is initialized 00000020:00000080:3.0:1789760789.879203:0:8284:0:(obd_config.c:826:class_setup()) finished setup of obd lustre-OST0001-osc-ffffa06c07e46000 (uuid b837c2fb-d1c1-4622-807e-a243f6fc2450) 00000020:01000000:3.0:1789760789.879210:0:8284:0:(obd_config.c:2004:class_config_llog_handler()) cmd cf00d, instance name: lustre-clilov-ffffa06c07e46000 00000020:00000080:3.0:1789760789.879213:0:8284:0:(obd_config.c:1472:class_process_config()) processing cmd: cf00d 00020000:01000000:3.0:1789760789.879222:0:8284:0:(lov_obd.c:481:lov_add_target()) uuid:lustre-OST0001_UUID idx:1 gen:1 active:1 00020000:01000000:3.0:1789760789.879226:0:8284:0:(lov_obd.c:510:lov_add_target()) lustre-clilov-ffffa06c07e46000: Adding target lustre-OST0001_UUID 00020000:01000000:3.0:1789760789.879229:0:8284:0:(lov_obd.c:532:lov_add_target()) idx=1 ltd_gen=1 ld_tgt_count=2 00000020:01000000:3.0:1789760789.879237:0:8284:0:(obd_config.c:1893:class_config_llog_handler()) Marker, inst_flg=0x2 mark_flg=0x2 00000020:00000080:3.0:1789760789.879240:0:8284:0:(obd_config.c:1472:class_process_config()) processing cmd: cf010 00000020:00000080:3.0:1789760789.879241:0:8284:0:(obd_config.c:1563:class_process_config()) marker 19 (0x2) lustre-OST0001 add osc 00000020:01000000:3.0:1789760789.879244:0:8284:0:(obd_config.c:1893:class_config_llog_handler()) Marker, inst_flg=0x0 mark_flg=0x1 00000020:00000080:3.0:1789760789.879247:0:8284:0:(obd_config.c:1472:class_process_config()) processing cmd: cf010 00000020:00000080:3.0:1789760789.879253:0:8284:0:(obd_config.c:1563:class_process_config()) marker 22 (0x1) lustre sys.timeout 00000020:00000080:3.0:1789760789.879257:0:8284:0:(obd_config.c:1472:class_process_config()) processing cmd: cf009 00000020:00000080:3.0:1789760789.879259:0:8284:0:(obd_config.c:1537:class_process_config()) changing lustre timeout from 20 to 20 00000020:01000000:3.0:1789760789.879261:0:8284:0:(obd_config.c:1893:class_config_llog_handler()) Marker, inst_flg=0x2 mark_flg=0x2 00000020:00000080:3.0:1789760789.879263:0:8284:0:(obd_config.c:1472:class_process_config()) processing cmd: cf010 00000020:00000080:3.0:1789760789.879265:0:8284:0:(obd_config.c:1563:class_process_config()) marker 22 (0x2) lustre sys.timeout 00000020:01000000:3.0:1789760789.879267:0:8284:0:(obd_config.c:1893:class_config_llog_handler()) Marker, inst_flg=0x0 mark_flg=0x1 00000020:00000080:3.0:1789760789.879269:0:8284:0:(obd_config.c:1472:class_process_config()) processing cmd: cf010 00000020:00000080:3.0:1789760789.879270:0:8284:0:(obd_config.c:1563:class_process_config()) marker 26 (0x1) lustre sys.jobid_var 00000020:00000080:3.0:1789760789.879273:0:8284:0:(obd_config.c:1472:class_process_config()) processing cmd: cf00f 00000100:00100000:3.0:1789760789.879314:0:8284:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872e300 pname:cluuid:pid:xid:nid:opc:job llog_process_th:02bc4b43-5e6f-4d7e-9224-80f5c789104f:8284:1876700024546432:192.168.203.107@tcp:502:llog_process_th.0 00000100:00100000:3.0:1789760789.879410:0:8284:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c108c96c0 going to sleep for 11 seconds 00000100:00100000:3.0:1789760789.886673:0:8284:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872e300 pname:cluuid:pid:xid:nid:opc:job llog_process_th:02bc4b43-5e6f-4d7e-9224-80f5c789104f:8284:1876700024546432:192.168.203.107@tcp:502:llog_process_th.0 00000020:01000000:3.0:1789760789.887079:0:8284:0:(obd_config.c:1893:class_config_llog_handler()) Marker, inst_flg=0x2 mark_flg=0x2 00000020:00000080:3.0:1789760789.887089:0:8284:0:(obd_config.c:1472:class_process_config()) processing cmd: cf010 00000020:00000080:3.0:1789760789.887091:0:8284:0:(obd_config.c:1563:class_process_config()) marker 26 (0x2) lustre sys.jobid_var 00000040:00080000:3.0:1789760789.887097:0:8284:0:(llog.c:862:llog_process_thread()) stop processing plain [0xa:0x3:0x0] index 42 count 42 00000020:01000000:1.0:1789760789.888375:0:8280:0:(obd_config.c:2145:class_config_parse_llog()) Processed log lustre-client gen 1-41 (rc=0) 10000000:01000000:1.0:1789760789.888406:0:8280:0:(mgc_request.c:1879:mgc_process_log()) MGC192.168.203.107@tcp: configuration from log 'lustre-client' succeeded (0). 00010000:00010000:1.0:1789760789.888427:0:8280:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.203.107@tcp lock: ffffa06c42174a00/0x59bc99bb619480d6 lrc: 3/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 4 type: PLN flags: 0x11000000000000 nid: local remote: 0x1daf3002fe9e094e expref: -99 pid: 8280 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1789760789.888439:0:8280:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: MGC192.168.203.107@tcp lock: ffffa06c42174a00/0x59bc99bb619480d6 lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 4 type: PLN flags: 0x11000000000000 nid: local remote: 0x1daf3002fe9e094e expref: -99 pid: 8280 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:1.0:1789760789.888447:0:8280:0:(mgc_request.c:1802:mgc_process_log()) Process log lustre-cliir-ffffa06c07e46000 from 1 10000000:01000000:1.0:1789760789.888450:0:8280:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-cliir (res 0x65727473756c) 00010000:00010000:1.0:1789760789.888482:0:8280:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.203.107@tcp lock: ffffa06c19d4cc00/0x59bc99bb61948100 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 3 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 8280 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1789760789.888489:0:8280:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.203.107@tcp lock: ffffa06c19d4cc00/0x59bc99bb61948100 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 3 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 8280 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1789760789.888500:0:8280:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.203.107@tcp lock: ffffa06c19d4cc00/0x59bc99bb61948100 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 3 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 8280 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1789760789.888521:0:8280:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2d180 pname:cluuid:pid:xid:nid:opc:job mount.lustre:02bc4b43-5e6f-4d7e-9224-80f5c789104f:8280:1876700024546560:192.168.203.107@tcp:101:mount.lustre.0 00000100:00100000:1.0:1789760789.888589:0:8280:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c04e32cc0 going to sleep for 11 seconds 00000100:00100000:3.0:1789760789.895123:0:8280:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2d180 pname:cluuid:pid:xid:nid:opc:job mount.lustre:02bc4b43-5e6f-4d7e-9224-80f5c789104f:8280:1876700024546560:192.168.203.107@tcp:101:mount.lustre.0 00010000:00010000:3.0:1789760789.895153:0:8280:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: MGC192.168.203.107@tcp lock: ffffa06c19d4cc00/0x59bc99bb61948100 lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 3 type: PLN flags: 0x11000000000000 nid: local remote: 0x1daf3002fe9e0955 expref: -99 pid: 8280 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1789760789.895166:0:8280:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: MGC192.168.203.107@tcp lock: ffffa06c19d4cc00/0x59bc99bb61948100 lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 3 type: PLN flags: 0x1000000000000 nid: local remote: 0x1daf3002fe9e0955 expref: -99 pid: 8280 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1789760789.895688:0:8280:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872e300 pname:cluuid:pid:xid:nid:opc:job mount.lustre:02bc4b43-5e6f-4d7e-9224-80f5c789104f:8280:1876700024546688:192.168.203.107@tcp:256:mount.lustre.0 00000100:00100000:3.0:1789760789.895785:0:8280:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c108c96c0 going to sleep for 11 seconds 00000100:00100000:3.0:1789760789.903363:0:8280:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872e300 pname:cluuid:pid:xid:nid:opc:job mount.lustre:02bc4b43-5e6f-4d7e-9224-80f5c789104f:8280:1876700024546688:192.168.203.107@tcp:256:mount.lustre.0 10000000:01000000:3.0:1789760789.904627:0:8280:0:(mgc_request.c:1879:mgc_process_log()) MGC192.168.203.107@tcp: configuration from log 'lustre-cliir' succeeded (0). 00010000:00010000:3.0:1789760789.904661:0:8280:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.203.107@tcp lock: ffffa06c19d4cc00/0x59bc99bb61948100 lrc: 3/1,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 3 type: PLN flags: 0x11000000000000 nid: local remote: 0x1daf3002fe9e0955 expref: -99 pid: 8280 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1789760789.904673:0:8280:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: MGC192.168.203.107@tcp lock: ffffa06c19d4cc00/0x59bc99bb61948100 lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 3 type: PLN flags: 0x11000000000000 nid: local remote: 0x1daf3002fe9e0955 expref: -99 pid: 8280 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:3.0:1789760789.904682:0:8280:0:(mgc_request.c:1802:mgc_process_log()) Process log params-ffffa06c07e46000 from 1 10000000:01000000:3.0:1789760789.904685:0:8280:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for params (res 0x736d61726170) 00010000:00010000:3.0:1789760789.904826:0:8280:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.203.107@tcp lock: ffffa06c42175200/0x59bc99bb61948107 lrc: 3/1,0 mode: --/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 3 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 8280 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1789760789.904836:0:8280:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.203.107@tcp lock: ffffa06c42175200/0x59bc99bb61948107 lrc: 3/1,0 mode: --/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 3 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 8280 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1789760789.904864:0:8280:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.203.107@tcp lock: ffffa06c42175200/0x59bc99bb61948107 lrc: 3/1,0 mode: --/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 3 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 8280 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1789760789.904886:0:8280:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872e300 pname:cluuid:pid:xid:nid:opc:job mount.lustre:02bc4b43-5e6f-4d7e-9224-80f5c789104f:8280:1876700024546816:192.168.203.107@tcp:101:mount.lustre.0 00000100:00100000:3.0:1789760789.904963:0:8280:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c108c96c0 going to sleep for 11 seconds 00000100:00100000:3.0:1789760789.908260:0:8280:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872e300 pname:cluuid:pid:xid:nid:opc:job mount.lustre:02bc4b43-5e6f-4d7e-9224-80f5c789104f:8280:1876700024546816:192.168.203.107@tcp:101:mount.lustre.0 00010000:00010000:3.0:1789760789.908286:0:8280:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: MGC192.168.203.107@tcp lock: ffffa06c42175200/0x59bc99bb61948107 lrc: 4/1,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 3 type: PLN flags: 0x11000000000000 nid: local remote: 0x1daf3002fe9e095c expref: -99 pid: 8280 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1789760789.908298:0:8280:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: MGC192.168.203.107@tcp lock: ffffa06c42175200/0x59bc99bb61948107 lrc: 4/1,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 3 type: PLN flags: 0x1000000000000 nid: local remote: 0x1daf3002fe9e095c expref: -99 pid: 8280 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1789760789.908366:0:8280:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872e300 pname:cluuid:pid:xid:nid:opc:job mount.lustre:02bc4b43-5e6f-4d7e-9224-80f5c789104f:8280:1876700024546944:192.168.203.107@tcp:501:mount.lustre.0 00000100:00100000:3.0:1789760789.908430:0:8280:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c108c96c0 going to sleep for 11 seconds 00000100:00100000:3.0:1789760789.912468:0:8280:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872e300 pname:cluuid:pid:xid:nid:opc:job mount.lustre:02bc4b43-5e6f-4d7e-9224-80f5c789104f:8280:1876700024546944:192.168.203.107@tcp:501:mount.lustre.0 00000100:00100000:3.0:1789760789.912529:0:8280:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872e300 pname:cluuid:pid:xid:nid:opc:job mount.lustre:02bc4b43-5e6f-4d7e-9224-80f5c789104f:8280:1876700024547072:192.168.203.107@tcp:503:mount.lustre.0 00000100:00100000:3.0:1789760789.912661:0:8280:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c108c96c0 going to sleep for 11 seconds 00000100:00100000:3.0:1789760789.925075:0:8280:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872e300 pname:cluuid:pid:xid:nid:opc:job mount.lustre:02bc4b43-5e6f-4d7e-9224-80f5c789104f:8280:1876700024547072:192.168.203.107@tcp:503:mount.lustre.0 00000100:00100000:0.0:1789760789.925712:0:8287:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d3b80 pname:cluuid:pid:xid:nid:opc:job llog_process_th:02bc4b43-5e6f-4d7e-9224-80f5c789104f:8287:1876700024547200:192.168.203.107@tcp:502:llog_process_th.0 00000100:00100000:0.0:1789760789.925811:0:8287:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c094aa600 going to sleep for 11 seconds 00000100:00100000:3.0:1789760789.934283:0:8287:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d3b80 pname:cluuid:pid:xid:nid:opc:job llog_process_th:02bc4b43-5e6f-4d7e-9224-80f5c789104f:8287:1876700024547200:192.168.203.107@tcp:502:llog_process_th.0 00000020:01000000:3.0:1789760789.934325:0:8287:0:(obd_config.c:1893:class_config_llog_handler()) Marker, inst_flg=0x0 mark_flg=0x1 00000020:00000080:3.0:1789760789.934335:0:8287:0:(obd_config.c:1472:class_process_config()) processing cmd: cf010 00000020:00000080:3.0:1789760789.934337:0:8287:0:(obd_config.c:1563:class_process_config()) marker 2 (0x1) general debug_raw_pointers 00000020:01000000:3.0:1789760789.934344:0:8287:0:(obd_config.c:2004:class_config_llog_handler()) cmd ce032, instance name: general-ffffa06c07e46000 00000020:00000080:3.0:1789760789.934348:0:8287:0:(obd_config.c:1472:class_process_config()) processing cmd: ce032 00000020:01000000:3.0:1789760789.934355:0:8287:0:(obd_config.c:1893:class_config_llog_handler()) Marker, inst_flg=0x2 mark_flg=0x2 00000020:00000080:3.0:1789760789.934357:0:8287:0:(obd_config.c:1472:class_process_config()) processing cmd: cf010 00000020:00000080:3.0:1789760789.934359:0:8287:0:(obd_config.c:1563:class_process_config()) marker 2 (0x2) general debug_raw_pointers 00000020:01000000:3.0:1789760789.934362:0:8287:0:(obd_config.c:1893:class_config_llog_handler()) Marker, inst_flg=0x0 mark_flg=0x1 00000020:00000080:3.0:1789760789.934364:0:8287:0:(obd_config.c:1472:class_process_config()) processing cmd: cf010 00000020:00000080:3.0:1789760789.934365:0:8287:0:(obd_config.c:1563:class_process_config()) marker 3 (0x1) general lod.*.mdt_hash 00000020:01000000:3.0:1789760789.934368:0:8287:0:(obd_config.c:2004:class_config_llog_handler()) cmd ce032, instance name: general-ffffa06c07e46000 00000020:00000080:3.0:1789760789.934371:0:8287:0:(obd_config.c:1472:class_process_config()) processing cmd: ce032 00000100:00100000:3.0:1789760789.934450:0:8287:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872e300 pname:cluuid:pid:xid:nid:opc:job llog_process_th:02bc4b43-5e6f-4d7e-9224-80f5c789104f:8287:1876700024547328:192.168.203.107@tcp:502:llog_process_th.0 00000100:00100000:3.0:1789760789.934533:0:8287:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c108c96c0 going to sleep for 11 seconds 00000100:00100000:3.0:1789760789.937569:0:8287:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872e300 pname:cluuid:pid:xid:nid:opc:job llog_process_th:02bc4b43-5e6f-4d7e-9224-80f5c789104f:8287:1876700024547328:192.168.203.107@tcp:502:llog_process_th.0 00000020:01000000:3.0:1789760789.937618:0:8287:0:(obd_config.c:1893:class_config_llog_handler()) Marker, inst_flg=0x2 mark_flg=0x2 00000020:00000080:3.0:1789760789.937626:0:8287:0:(obd_config.c:1472:class_process_config()) processing cmd: cf010 00000020:00000080:3.0:1789760789.937629:0:8287:0:(obd_config.c:1563:class_process_config()) marker 3 (0x2) general lod.*.mdt_hash 00000040:00080000:3.0:1789760789.937634:0:8287:0:(llog.c:862:llog_process_thread()) stop processing plain [0xa:0x2:0x0] index 7 count 7 00000020:01000000:0.0:1789760789.937946:0:8280:0:(obd_config.c:2145:class_config_parse_llog()) Processed log params gen 1-6 (rc=0) 10000000:01000000:0.0:1789760789.937966:0:8280:0:(mgc_request.c:1879:mgc_process_log()) MGC192.168.203.107@tcp: configuration from log 'params' succeeded (0). 00010000:00010000:0.0:1789760789.937982:0:8280:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.203.107@tcp lock: ffffa06c42175200/0x59bc99bb61948107 lrc: 3/1,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 3 type: PLN flags: 0x11000000000000 nid: local remote: 0x1daf3002fe9e095c expref: -99 pid: 8280 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1789760789.937994:0:8280:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: MGC192.168.203.107@tcp lock: ffffa06c42175200/0x59bc99bb61948107 lrc: 2/0,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 3 type: PLN flags: 0x11000000000000 nid: local remote: 0x1daf3002fe9e095c expref: -99 pid: 8280 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000080:01000000:0.0:1789760789.938012:0:8280:0:(llite_lib.c:1614:ll_fill_super()) Found profile lustre-client: mdc=lustre-clilmv osc=lustre-clilov 00000020:00000080:0.0:1789760789.938036:0:8280:0:(genops.c:1385:class_connect()) connect: client b837c2fb-d1c1-4622-807e-a243f6fc2450, cookie 0x59bc99bb6194810e 00800000:01000000:0.0:1789760789.938055:0:8280:0:(lmv_obd.c:515:lmv_check_connect()) Time to connect b837c2fb-d1c1-4622-807e-a243f6fc2450 to lustre-clilmv-ffffa06c07e46000 00800000:01000000:0.0:1789760789.938063:0:8280:0:(lmv_obd.c:338:lmv_connect_mdc()) connect to lustre-MDT0000-mdc-ffffa06c07e46000(b837c2fb-d1c1-4622-807e-a243f6fc2450) - lustre-MDT0000_UUID, b837c2fb-d1c1-4622-807e-a243f6fc2450 00000020:00000080:0.0:1789760789.938070:0:8280:0:(genops.c:1385:class_connect()) connect: client b837c2fb-d1c1-4622-807e-a243f6fc2450, cookie 0x59bc99bb61948115 00000100:00080000:0.0:1789760789.938078:0:8280:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 0 00000100:00080000:0.0:1789760789.938085:0:8280:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789760789.938095:0:8280:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 134217728, lost: 0. 00000100:00100000:0.0:1789760789.938117:0:8280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d2680 x1876700024547456/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789760789.938144:0:8280:0:(pinger.c:373:ptlrpc_pinger_add_import()) adding pingable import b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID 00800000:01000000:0.0:1789760789.938229:0:8280:0:(lmv_obd.c:392:lmv_connect_mdc()) Connected to lustre-MDT0000-mdc-ffffa06c07e46000(b837c2fb-d1c1-4622-807e-a243f6fc2450) successfully (5) 00800000:00000004:0.0:1789760789.938232:0:8280:0:(obd_class.h:1056:obd_statfs_async()) lustre-MDT0000-mdc-ffffa06c07e46000: age -772, max_age 0 00000080:00000004:0.0:1789760789.938270:0:8280:0:(obd_class.h:1090:obd_statfs()) lustre-clilmv-ffffa06c07e46000: age -772, max_age 227 00800000:00000004:0.0:1789760789.938275:0:8280:0:(obd_class.h:1090:obd_statfs()) lustre-MDT0000-mdc-ffffa06c07e46000: age -772, max_age 227 00000100:00080000:0.0:1789760789.938293:0:8280:0:(client.c:1845:ptlrpc_send_new_req()) @@@ req waiting for recovery: (FULL != CONNECTING) req@ffffa06c091d2a00 x1876700024547712/t0(0) o41->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 224/368 e 0 to 0 dl 0 ref 2 fl Rpc:WQU/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760789.938302:0:8280:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c094aa000 going to sleep for 0 seconds 00000100:00100000:1.0:1789760789.939884:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024547456:192.168.203.107@tcp:38:mount.lustre.0 00000100:00080000:1.0:1789760789.940015:0:2342:0:(client.c:1845:ptlrpc_send_new_req()) @@@ req waiting for recovery: (FULL != CONNECTING) req@ffffa06c091d3480 x1876700024547584/t0(0) o41->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 224/368 e 0 to 0 dl 0 ref 1 fl Rpc:WQU/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789760789.940032:0:2342:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1789760789.940039:0:2342:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2342:1876700024547840:192.168.203.107@tcp:400:kworker.0 00000100:00100000:2.0:1789760789.950122:0:2340:0:(client.c:3125:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffffa06c07e46000: committing for last_committed 0 gen 1 00000100:00100000:0.0:1789760789.950126:0:2342:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2342:1876700024547840:192.168.203.107@tcp:400:kworker.0 00000100:00080000:2.0:1789760789.950135:0:2340:0:(import.c:1132:ptlrpc_connect_interpret()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to target with instance 1 00000100:00080000:2.0:1789760789.950147:0:2340:0:(import.c:956:ptlrpc_connect_set_flags()) lustre-MDT0000-mdc-ffffa06c07e46000: Resetting ns_connect_flags to server flags: 0xae7a7e7be344d3b8 00000100:00080000:2.0:1789760789.950170:0:2340:0:(import.c:1213:ptlrpc_connect_interpret()) connected to replayable target: lustre-MDT0000_UUID 00000100:00080000:2.0:1789760789.950175:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to FULL 00000100:00080000:3.0:1789760789.950388:0:2340:0:(recover.c:191:ptlrpc_wake_delayed()) @@@ waking (set 0000000086be59e9): req@ffffa06c091d2a00 x1876700024547712/t0(0) o41->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 224/368 e 0 to 0 dl 0 ref 2 fl Rpc:WQU/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789760789.950427:0:2340:0:(recover.c:191:ptlrpc_wake_delayed()) @@@ waking (set 0000000063396333): req@ffffa06c091d3480 x1876700024547584/t0(0) o41->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 224/368 e 0 to 0 dl 0 ref 1 fl Rpc:WQU/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789760789.950439:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024547456:192.168.203.107@tcp:38:mount.lustre.0 00000100:00100000:0.0:1789760789.951204:0:8280:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c094aa000 going to sleep for 11 seconds 00000100:00100000:0.0:1789760789.953220:0:2342:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:b837c2fb-d1c1-4622-807e-a243f6fc2450:0:1876700024547584:192.168.203.107@tcp:41:mount.lustre.0 00000100:00100000:2.0:1789760789.957347:0:8280:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d2a00 pname:cluuid:pid:xid:nid:opc:job mount.lustre:b837c2fb-d1c1-4622-807e-a243f6fc2450:8280:1876700024547712:192.168.203.107@tcp:41:mount.lustre.0 00800000:00000004:2.0:1789760789.957386:0:8280:0:(obd_class.h:1116:obd_statfs()) lustre-MDT0000-mdc-ffffa06c07e46000: update ffffa06c181e5178 cache blocks 317407/353529 objects 1023725/1024000 00000080:00000004:2.0:1789760789.957394:0:8280:0:(obd_class.h:1116:obd_statfs()) lustre-clilmv-ffffa06c07e46000: update ffffa06c181e37c0 cache blocks 317407/353529 objects 1023725/1024000 00000100:00100000:2.0:1789760789.957432:0:8280:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e53800 pname:cluuid:pid:xid:nid:opc:job mount.lustre:b837c2fb-d1c1-4622-807e-a243f6fc2450:8280:1876700024547968:192.168.203.107@tcp:40:mount.lustre.0 00000100:00100000:2.0:1789760789.957520:0:8280:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c04e42a80 going to sleep for 11 seconds 00000100:00100000:2.0:1789760789.963840:0:8280:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e53800 pname:cluuid:pid:xid:nid:opc:job mount.lustre:b837c2fb-d1c1-4622-807e-a243f6fc2450:8280:1876700024547968:192.168.203.107@tcp:40:mount.lustre.0 00000080:00000004:2.0:1789760789.963898:0:8280:0:(llite_lib.c:614:client_common_fill_super()) rootfid [0x200000007:0x1:0x0] 00000080:00100000:2.0:1789760789.963917:0:8280:0:(llite_lib.c:775:client_common_fill_super()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 2146304 00020000:01000000:2.0:1789760789.963921:0:8280:0:(lov_obd.c:186:lov_connect()) connect #0 00000020:00000080:2.0:1789760789.963934:0:8280:0:(genops.c:1385:class_connect()) connect: client b837c2fb-d1c1-4622-807e-a243f6fc2450, cookie 0x59bc99bb6194811c 00000020:00000080:2.0:1789760789.963951:0:8280:0:(genops.c:1385:class_connect()) connect: client LOV_OSC_UUID, cookie 0x59bc99bb61948123 00000100:00080000:2.0:1789760789.963958:0:8280:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 0 00000100:00080000:2.0:1789760789.963968:0:8280:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffffa06c07e46000: import ffffa06c09133000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789760789.963980:0:8280:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 134217728, lost: 0. 00000100:00100000:2.0:1789760789.964007:0:8280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e53800 x1876700024548096/t0(0) o8->lustre-OST0000-osc-ffffa06c07e46000@192.168.203.107@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789760789.964036:0:8280:0:(pinger.c:373:ptlrpc_pinger_add_import()) adding pingable import b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-OST0000_UUID 00020000:01000000:2.0:1789760789.964059:0:8280:0:(lov_obd.c:159:lov_connect_osc()) Connected tgt idx 0 lustre-OST0000_UUID (lustre-OST0000-osc-ffffa06c07e46000) active 00000100:00100000:3.0:1789760789.964072:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e53800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024548096:192.168.203.107@tcp:8:mount.lustre.0 00000020:00000080:2.0:1789760789.964749:0:8280:0:(genops.c:1385:class_connect()) connect: client LOV_OSC_UUID, cookie 0x59bc99bb6194812a 00000100:00080000:2.0:1789760789.964757:0:8280:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 0 00000100:00080000:2.0:1789760789.964773:0:8280:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffffa06c07e46000: import ffffa06c09134000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789760789.964782:0:8280:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 134217728, lost: 0. 00000100:00100000:2.0:1789760789.964797:0:8280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50e00 x1876700024548224/t0(0) o8->lustre-OST0001-osc-ffffa06c07e46000@192.168.203.107@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789760789.964837:0:8280:0:(pinger.c:373:ptlrpc_pinger_add_import()) adding pingable import b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-OST0001_UUID 00020000:01000000:2.0:1789760789.964870:0:8280:0:(lov_obd.c:159:lov_connect_osc()) Connected tgt idx 1 lustre-OST0001_UUID (lustre-OST0001-osc-ffffa06c07e46000) active 00000100:00100000:2.0:1789760789.964919:0:8280:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50380 pname:cluuid:pid:xid:nid:opc:job mount.lustre:b837c2fb-d1c1-4622-807e-a243f6fc2450:8280:1876700024548352:192.168.203.107@tcp:33:mount.lustre.0 00000100:00100000:2.0:1789760789.965003:0:8280:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c04e42a80 going to sleep for 11 seconds 00000100:00100000:3.0:1789760789.967526:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024548224:192.168.203.107@tcp:8:mount.lustre.0 00010000:00080000:3.0:1789760789.969777:0:2340:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61549) req@ffffa06c07e53800 x1876700024548096/t0(0) o8->lustre-OST0000-osc-ffffa06c07e46000@192.168.203.107@tcp:28/4 lens 520/416 e 0 to 0 dl 1789760799 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789760789.969791:0:2340:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffffa06c07e46000: committing for last_committed 0 gen 1 00000100:00080000:3.0:1789760789.969801:0:2340:0:(import.c:1132:ptlrpc_connect_interpret()) lustre-OST0000-osc-ffffa06c07e46000: connect to target with instance 1 00000100:00080000:3.0:1789760789.969811:0:2340:0:(import.c:956:ptlrpc_connect_set_flags()) lustre-OST0000-osc-ffffa06c07e46000: Resetting ns_connect_flags to server flags: 0xa0427af2e3440038 00000100:00080000:3.0:1789760789.969823:0:2340:0:(import.c:1213:ptlrpc_connect_interpret()) connected to replayable target: lustre-OST0000_UUID 00000100:00080000:3.0:1789760789.969827:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09133000 lustre-OST0000_UUID: changing import state from CONNECTING to FULL 00020000:01000000:3.0:1789760789.969833:0:2340:0:(lov_obd.c:398:lov_set_osc_active()) Marking OSC lustre-OST0000_UUID active 00000080:00000004:3.0:1789760789.969837:0:2340:0:(lcommon_misc.c:94:cl_ocd_update()) Changing connect_flags: 0xa0467af2e3640038 -> 0xa0427af2e3440038 00000080:00080000:2.0:1789760789.970593:0:2340:0:(lcommon_misc.c:56:cl_init_ea_size()) updating def/max_easize: 72/65536 00000100:00100000:2.0:1789760789.970610:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e53800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024548096:192.168.203.107@tcp:8:mount.lustre.0 00000100:00100000:2.0:1789760789.974020:0:8280:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50380 pname:cluuid:pid:xid:nid:opc:job mount.lustre:b837c2fb-d1c1-4622-807e-a243f6fc2450:8280:1876700024548352:192.168.203.107@tcp:33:mount.lustre.0 00000080:00200000:2.0:1789760789.974206:0:8280:0:(llite_lib.c:3356:ll_read_inode2()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c035742c8) 00000080:00200000:2.0:1789760789.974214:0:8280:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 4096 00000080:02000400:2.0:1789760789.974277:0:8280:0:(llite_lib.c:1653:ll_fill_super()) Mounted lustre-client - version 2.17.57_103_g082f77f 00010000:00080000:3.0:1789760789.977921:0:2340:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61549) req@ffffa06c07e50e00 x1876700024548224/t0(0) o8->lustre-OST0001-osc-ffffa06c07e46000@192.168.203.107@tcp:28/4 lens 520/416 e 0 to 0 dl 1789760799 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789760789.977939:0:2340:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0001-osc-ffffa06c07e46000: committing for last_committed 0 gen 1 00000100:00080000:3.0:1789760789.977949:0:2340:0:(import.c:1132:ptlrpc_connect_interpret()) lustre-OST0001-osc-ffffa06c07e46000: connect to target with instance 1 00000100:00080000:3.0:1789760789.978122:0:2340:0:(import.c:956:ptlrpc_connect_set_flags()) lustre-OST0001-osc-ffffa06c07e46000: Resetting ns_connect_flags to server flags: 0xa0427af2e3440038 00000100:00080000:3.0:1789760789.978134:0:2340:0:(import.c:1213:ptlrpc_connect_interpret()) connected to replayable target: lustre-OST0001_UUID 00000100:00080000:3.0:1789760789.978138:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09134000 lustre-OST0001_UUID: changing import state from CONNECTING to FULL 00020000:01000000:3.0:1789760789.978145:0:2340:0:(lov_obd.c:398:lov_set_osc_active()) Marking OSC lustre-OST0001_UUID active 00000080:00000004:3.0:1789760789.978172:0:2340:0:(lcommon_misc.c:94:cl_ocd_update()) Changing connect_flags: 0xa0427af2e3440038 -> 0xa0427af2e3440038 00000080:00080000:3.0:1789760789.978177:0:2340:0:(lcommon_misc.c:56:cl_init_ea_size()) updating def/max_easize: 72/65536 00000100:00100000:3.0:1789760789.978189:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024548224:192.168.203.107@tcp:8:mount.lustre.0 00000080:00000004:2.0:1789760789.983477:0:8280:0:(super25.c:182:lustre_fill_super()) 192.168.203.107@tcp:/lustre: Mount complete 00010000:00010000:1.0:1789760790.141689:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789760791.159082:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789760792.183158:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789760793.208602:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789760794.231785:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000001:02000400:0.0:1789760794.414197:0:8543:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: oleg307-client.virtnet: executing check_config_client /mnt/lustre 00000100:00100000:3.0:1789760794.999666:0:2343:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2343:1876700024548480:192.168.203.107@tcp:400:kworker.0 00000100:00100000:3.0:1789760794.999818:0:2343:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:b837c2fb-d1c1-4622-807e-a243f6fc2450:2343:1876700024548736:192.168.203.107@tcp:400:kworker.0 00000100:00100000:3.0:1789760795.001014:0:2343:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e52680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:b837c2fb-d1c1-4622-807e-a243f6fc2450:2343:1876700024548992:192.168.203.107@tcp:400:kworker.0 00000100:00100000:3.0:1789760795.001063:0:2343:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [1->0] 00000100:00100000:3.0:1789760795.001071:0:2343:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e53800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2343:1876700024548608:192.168.203.107@tcp:400:kworker.0 00000100:00100000:3.0:1789760795.001097:0:2343:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e53b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:b837c2fb-d1c1-4622-807e-a243f6fc2450:2343:1876700024548864:192.168.203.107@tcp:400:kworker.0 00010000:00080000:3.0:1789760795.009408:0:2343:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61549) req@ffffa06c07e53b80 x1876700024548864/t0(0) o400->lustre-OST0000-osc-ffffa06c07e46000@192.168.203.107@tcp:28/4 lens 224/224 e 0 to 0 dl 1789760811 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789760795.009433:0:2343:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e53b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:b837c2fb-d1c1-4622-807e-a243f6fc2450:2343:1876700024548864:192.168.203.107@tcp:400:kworker.0 00000100:00100000:3.0:1789760795.009448:0:2343:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2343:1876700024548480:192.168.203.107@tcp:400:kworker.0 00000100:00100000:3.0:1789760795.009461:0:2343:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:b837c2fb-d1c1-4622-807e-a243f6fc2450:2343:1876700024548736:192.168.203.107@tcp:400:kworker.0 00010000:00080000:3.0:1789760795.009474:0:2343:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61549) req@ffffa06c07e52680 x1876700024548992/t0(0) o400->lustre-OST0001-osc-ffffa06c07e46000@192.168.203.107@tcp:28/4 lens 224/224 e 0 to 0 dl 1789760811 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789760795.009483:0:2343:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e52680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:b837c2fb-d1c1-4622-807e-a243f6fc2450:2343:1876700024548992:192.168.203.107@tcp:400:kworker.0 00000100:00100000:3.0:1789760795.009719:0:2343:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e53800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2343:1876700024548608:192.168.203.107@tcp:400:kworker.0 00010000:00010000:0.0:1789760795.256460:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789760796.279367:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789760797.303088:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789760798.328538:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789760799.352470:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789760800.119194:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e53480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2344:1876700024549120:192.168.203.107@tcp:400:kworker.0 00000100:00100000:2.0:1789760800.119216:0:2343:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e52a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2343:1876700024549248:192.168.203.107@tcp:400:kworker.0 00000100:00100000:2.0:1789760800.119328:0:2343:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e52d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:b837c2fb-d1c1-4622-807e-a243f6fc2450:2343:1876700024549504:192.168.203.107@tcp:400:kworker.0 00000100:00100000:3.0:1789760800.119332:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:b837c2fb-d1c1-4622-807e-a243f6fc2450:2344:1876700024549376:192.168.203.107@tcp:400:kworker.0 00000100:00100000:2.0:1789760800.119374:0:2343:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1789760800.119385:0:2343:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:b837c2fb-d1c1-4622-807e-a243f6fc2450:2343:1876700024549632:192.168.203.107@tcp:400:kworker.0 00000100:00100000:3.0:1789760800.121289:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e53480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2344:1876700024549120:192.168.203.107@tcp:400:kworker.0 00000100:00100000:3.0:1789760800.122581:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:b837c2fb-d1c1-4622-807e-a243f6fc2450:2344:1876700024549376:192.168.203.107@tcp:400:kworker.0 00010000:00080000:2.0:1789760800.123218:0:2343:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61549) req@ffffa06c07e50700 x1876700024549632/t0(0) o400->lustre-OST0001-osc-ffffa06c07e46000@192.168.203.107@tcp:28/4 lens 224/224 e 0 to 0 dl 1789760816 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789760800.123231:0:2343:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:b837c2fb-d1c1-4622-807e-a243f6fc2450:2343:1876700024549632:192.168.203.107@tcp:400:kworker.0 00000100:00100000:2.0:1789760800.123243:0:2343:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e52a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2343:1876700024549248:192.168.203.107@tcp:400:kworker.0 00010000:00080000:2.0:1789760800.123269:0:2343:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61549) req@ffffa06c07e52d80 x1876700024549504/t0(0) o400->lustre-OST0000-osc-ffffa06c07e46000@192.168.203.107@tcp:28/4 lens 224/224 e 0 to 0 dl 1789760816 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789760800.123278:0:2343:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e52d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:b837c2fb-d1c1-4622-807e-a243f6fc2450:2343:1876700024549504:192.168.203.107@tcp:400:kworker.0 00010000:00010000:1.0:1789760800.374990:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789760801.399190:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789760802.424050:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000001:02000400:3.0:1789760802.760175:0:9197:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: Using TIMEOUT=20 00010000:00010000:0.0:1789760803.447012:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789760803.868402:0:2334:0:(events.c:340:request_in_callback()) peer: 12345-192.168.203.107@tcp (source: 12345-192.168.203.107@tcp) 00000100:00100000:1.0:1789760803.868468:0:5418:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1876700049745792 00000100:00100000:1.0:1789760803.868479:0:5418:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1876700049745792 00000100:00100000:1.0:1789760803.868530:0:5418:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.107@tcp, seq: 3 00000100:00100000:1.0:1789760803.868538:0:5418:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffffa06c091d3480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:02bc4b43-5e6f-4d7e-9224-80f5c789104f+3:11702:x1876700049745792:12345-192.168.203.107@tcp:104: 00010000:00010000:1.0:1789760803.868555:0:5418:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: MGC192.168.203.107@tcp lock: ffffa06c42175200/0x59bc99bb61948107 lrc: 2/0,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 3 type: PLN flags: 0x1400000000000 nid: local remote: 0x1daf3002fe9e095c expref: -99 pid: 8280 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1789760803.868596:0:2333:0:(events.c:340:request_in_callback()) peer: 12345-192.168.203.107@tcp (source: 12345-192.168.203.107@tcp) 00010000:00010000:1.0:1789760803.868631:0:5418:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1789760803.868639:0:5418:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffffa06c091d3480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:02bc4b43-5e6f-4d7e-9224-80f5c789104f+3:11702:x1876700049745792:12345-192.168.203.107@tcp:104: Request processed in 102us (249us total) trans 0 rc 0/0 00000100:00100000:1.0:1789760803.868647:0:5418:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.107@tcp, seq: 3 00000100:00100000:1.0:1789760803.868676:0:5418:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1876700049746048 00000100:00100000:1.0:1789760803.868680:0:5418:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1876700049746048 00010000:00010000:2.0:1789760803.868682:0:5428:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1789760803.868690:0:5428:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.203.107@tcp lock: ffffa06c42175200/0x59bc99bb61948107 lrc: 2/0,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 3 type: PLN flags: 0x1400000000000 nid: local remote: 0x1daf3002fe9e095c expref: -99 pid: 8280 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1789760803.868698:0:5428:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffffa06c42175200 already unused, calling callback (ffffffffc13cfc20) 00000100:00100000:1.0:1789760803.868701:0:5418:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.107@tcp, seq: 4 10000000:00010000:2.0:1789760803.868704:0:5428:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.203.107@tcp lock: ffffa06c42175200/0x59bc99bb61948107 lrc: 2/0,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 3 type: PLN flags: 0x1400400000000 nid: local remote: 0x1daf3002fe9e095c expref: -99 pid: 8280 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1789760803.868708:0:5417:0:(service.c:3462:ptlrpc_start_thread()) ldlm_cbd[0] started 2 min 2 max 34 00010000:00010000:2.0:1789760803.868712:0:5428:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.203.107@tcp lock: ffffa06c42175200/0x59bc99bb61948107 lrc: 3/0,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 3 type: PLN flags: 0x1408400000000 nid: local remote: 0x1daf3002fe9e095c expref: -99 pid: 8280 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1789760803.868713:0:5418:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffffa06c091d1c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:02bc4b43-5e6f-4d7e-9224-80f5c789104f+3:11702:x1876700049746048:12345-192.168.203.107@tcp:104: 00000100:00100000:0.0:1789760803.868719:0:5417:0:(service.c:3521:ptlrpc_start_thread()) starting thread 'ldlm_cb00_002' 00010000:00010000:1.0:1789760803.868725:0:5418:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: MGC192.168.203.107@tcp lock: ffffa06c01512e00/0x59bc99bb619480cf lrc: 2/0,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 4 type: PLN flags: 0x1400000000000 nid: local remote: 0x1daf3002fe9e0939 expref: -99 pid: 5431 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00010000:2.0:1789760803.868726:0:5428:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.203.107@tcp lock: ffffa06c42175200/0x59bc99bb61948107 lrc: 3/0,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 4 type: PLN flags: 0x1409400000000 nid: local remote: 0x1daf3002fe9e095c expref: -99 pid: 8280 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:2.0:1789760803.868742:0:5428:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x736d61726170:0x3:0x0].0x0 (params) 00010000:00010000:1.0:1789760803.868775:0:5418:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00010000:00010000:2.0:1789760803.868781:0:5428:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: MGC192.168.203.107@tcp lock: ffffa06c42175200/0x59bc99bb61948107 lrc: 2/0,0 mode: --/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 3 type: PLN flags: 0x5c09400000020 nid: local remote: 0x1daf3002fe9e095c expref: -99 pid: 8280 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1789760803.868782:0:5418:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffffa06c091d1c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:02bc4b43-5e6f-4d7e-9224-80f5c789104f+3:11702:x1876700049746048:12345-192.168.203.107@tcp:104: Request processed in 72us (187us total) trans 0 rc 0/0 00000100:00100000:1.0:1789760803.868795:0:5418:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.107@tcp, seq: 4 00010000:00010000:2.0:1789760803.868796:0:5428:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1789760803.868812:0:5427:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1789760803.868818:0:5428:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.203.107@tcp lock: ffffa06c42175200/0x59bc99bb61948107 lrc: 1/0,0 mode: --/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 3 type: PLN flags: 0x5c09400000020 nid: local remote: 0x1daf3002fe9e095c expref: -99 pid: 8280 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1789760803.868825:0:5428:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.203.107@tcp lock: ffffa06c42175200/0x59bc99bb61948107 lrc: 0/0,0 mode: --/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 3 type: PLN flags: 0x5c09400000020 nid: local remote: 0x1daf3002fe9e095c expref: -99 pid: 8280 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1789760803.868840:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789760803.868892:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789760803.868894:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789760803.868909:0:2343:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e52d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2343:1876700024549760:192.168.203.107@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1789760803.869048:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789760803.869051:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789760803.869053:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789760803.869150:0:5427:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.203.107@tcp lock: ffffa06c01512e00/0x59bc99bb619480cf lrc: 2/0,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x1400000000000 nid: local remote: 0x1daf3002fe9e0939 expref: -99 pid: 5431 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1789760803.869157:0:5427:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffffa06c01512e00 already unused, calling callback (ffffffffc13cfc20) 10000000:00010000:3.0:1789760803.869159:0:5427:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.203.107@tcp lock: ffffa06c01512e00/0x59bc99bb619480cf lrc: 2/0,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x1400400000000 nid: local remote: 0x1daf3002fe9e0939 expref: -99 pid: 5431 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1789760803.869165:0:5427:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.203.107@tcp lock: ffffa06c01512e00/0x59bc99bb619480cf lrc: 3/0,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x1408400000000 nid: local remote: 0x1daf3002fe9e0939 expref: -99 pid: 5431 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00010000:3.0:1789760803.869169:0:5427:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.203.107@tcp lock: ffffa06c01512e00/0x59bc99bb619480cf lrc: 3/0,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x1409400000000 nid: local remote: 0x1daf3002fe9e0939 expref: -99 pid: 5431 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:3.0:1789760803.869173:0:5427:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x736d61726170:0x3:0x0].0x0 (params) 00010000:00010000:3.0:1789760803.869193:0:5427:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: MGC192.168.203.107@tcp lock: ffffa06c01512e00/0x59bc99bb619480cf lrc: 2/0,0 mode: --/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x5c09400000020 nid: local remote: 0x1daf3002fe9e0939 expref: -99 pid: 5431 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1789760803.869199:0:5427:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:3.0:1789760803.869215:0:5427:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.203.107@tcp lock: ffffa06c01512e00/0x59bc99bb619480cf lrc: 1/0,0 mode: --/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x5c09400000020 nid: local remote: 0x1daf3002fe9e0939 expref: -99 pid: 5431 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1789760803.869223:0:5427:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.203.107@tcp lock: ffffa06c01512e00/0x59bc99bb619480cf lrc: 0/0,0 mode: --/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x5c09400000020 nid: local remote: 0x1daf3002fe9e0939 expref: -99 pid: 5431 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1789760803.869230:0:2344:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2344:1876700024549888:192.168.203.107@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1789760803.869254:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789760803.869257:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789760803.869260:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789760803.869693:0:2343:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e52d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2343:1876700024549760:192.168.203.107@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1789760803.869983:0:2344:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2344:1876700024549888:192.168.203.107@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1789760804.470976:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 10000000:01000000:0.0:1789760804.982977:0:5431:0:(mgc_request.c:611:do_requeue()) updating log params 10000000:01000000:0.0:1789760804.982985:0:5431:0:(mgc_request.c:1802:mgc_process_log()) Process log params-ffffa06c07e46000 from 7 10000000:01000000:0.0:1789760804.982990:0:5431:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for params (res 0x736d61726170) 00010000:00010000:0.0:1789760804.983044:0:5431:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.203.107@tcp lock: ffffa06c110cb800/0x59bc99bb61948131 lrc: 3/1,0 mode: --/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 5431 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1789760804.983054:0:5431:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.203.107@tcp lock: ffffa06c110cb800/0x59bc99bb61948131 lrc: 3/1,0 mode: --/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 5431 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1789760804.983084:0:5431:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.203.107@tcp lock: ffffa06c110cb800/0x59bc99bb61948131 lrc: 3/1,0 mode: --/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 5431 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1789760804.983110:0:5431:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d2d80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:02bc4b43-5e6f-4d7e-9224-80f5c789104f:5431:1876700024550016:192.168.203.107@tcp:101:ll_cfg_requeue.0 00000100:00100000:0.0:1789760804.983219:0:5431:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c10d4f480 going to sleep for 11 seconds 00000100:00100000:2.0:1789760804.984313:0:5431:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d2d80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:02bc4b43-5e6f-4d7e-9224-80f5c789104f:5431:1876700024550016:192.168.203.107@tcp:101:ll_cfg_requeue.0 00010000:00010000:2.0:1789760804.984331:0:5431:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: MGC192.168.203.107@tcp lock: ffffa06c110cb800/0x59bc99bb61948131 lrc: 4/1,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x1daf3002fe9e097f expref: -99 pid: 5431 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1789760804.984341:0:5431:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: MGC192.168.203.107@tcp lock: ffffa06c110cb800/0x59bc99bb61948131 lrc: 4/1,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x1daf3002fe9e097f expref: -99 pid: 5431 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1789760804.984397:0:5431:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e52a00 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:02bc4b43-5e6f-4d7e-9224-80f5c789104f:5431:1876700024550144:192.168.203.107@tcp:501:ll_cfg_requeue.0 00000100:00100000:2.0:1789760804.984438:0:5431:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c055ca000 going to sleep for 11 seconds 00000100:00100000:2.0:1789760804.985219:0:5431:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e52a00 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:02bc4b43-5e6f-4d7e-9224-80f5c789104f:5431:1876700024550144:192.168.203.107@tcp:501:ll_cfg_requeue.0 00000100:00100000:2.0:1789760804.985266:0:5431:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e52a00 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:02bc4b43-5e6f-4d7e-9224-80f5c789104f:5431:1876700024550272:192.168.203.107@tcp:503:ll_cfg_requeue.0 00000100:00100000:2.0:1789760804.985333:0:5431:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c055ca000 going to sleep for 11 seconds 00000100:00100000:2.0:1789760804.986036:0:5431:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e52a00 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:02bc4b43-5e6f-4d7e-9224-80f5c789104f:5431:1876700024550272:192.168.203.107@tcp:503:ll_cfg_requeue.0 00000040:00080000:1.0:1789760804.986895:0:9471:0:(llog.c:862:llog_process_thread()) stop processing plain [0xa:0x2:0x0] index 64768 count 7 00000020:01000000:0.0:1789760804.986999:0:5431:0:(obd_config.c:2145:class_config_parse_llog()) Processed log params gen 7-6 (rc=0) 10000000:01000000:0.0:1789760804.987030:0:5431:0:(mgc_request.c:1879:mgc_process_log()) MGC192.168.203.107@tcp: configuration from log 'params' succeeded (0). 00010000:00010000:0.0:1789760804.987038:0:5431:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.203.107@tcp lock: ffffa06c110cb800/0x59bc99bb61948131 lrc: 3/1,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x1daf3002fe9e097f expref: -99 pid: 5431 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1789760804.987048:0:5431:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: MGC192.168.203.107@tcp lock: ffffa06c110cb800/0x59bc99bb61948131 lrc: 2/0,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x1daf3002fe9e097f expref: -99 pid: 5431 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:0.0:1789760804.987058:0:5431:0:(mgc_request.c:611:do_requeue()) updating log params 10000000:01000000:0.0:1789760804.987061:0:5431:0:(mgc_request.c:1802:mgc_process_log()) Process log params-ffffa06c18a1e000 from 7 10000000:01000000:0.0:1789760804.987063:0:5431:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for params (res 0x736d61726170) 00010000:00010000:0.0:1789760804.987085:0:5431:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.203.107@tcp lock: ffffa06c110ca800/0x59bc99bb61948138 lrc: 3/1,0 mode: --/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 3 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 5431 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1789760804.987091:0:5431:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.203.107@tcp lock: ffffa06c110ca800/0x59bc99bb61948138 lrc: 3/1,0 mode: --/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 3 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 5431 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1789760804.987097:0:5431:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.203.107@tcp lock: ffffa06c110ca800/0x59bc99bb61948138 lrc: 3/1,0 mode: --/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 3 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 5431 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1789760804.987109:0:5431:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0700 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:02bc4b43-5e6f-4d7e-9224-80f5c789104f:5431:1876700024550400:192.168.203.107@tcp:101:ll_cfg_requeue.0 00000100:00100000:0.0:1789760804.987151:0:5431:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c10d4f240 going to sleep for 11 seconds 00000100:00100000:3.0:1789760804.988113:0:5431:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0700 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:02bc4b43-5e6f-4d7e-9224-80f5c789104f:5431:1876700024550400:192.168.203.107@tcp:101:ll_cfg_requeue.0 00010000:00010000:3.0:1789760804.988134:0:5431:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: MGC192.168.203.107@tcp lock: ffffa06c110ca800/0x59bc99bb61948138 lrc: 4/1,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 3 type: PLN flags: 0x11000000000000 nid: local remote: 0x1daf3002fe9e0986 expref: -99 pid: 5431 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1789760804.988146:0:5431:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: MGC192.168.203.107@tcp lock: ffffa06c110ca800/0x59bc99bb61948138 lrc: 4/1,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 3 type: PLN flags: 0x1000000000000 nid: local remote: 0x1daf3002fe9e0986 expref: -99 pid: 5431 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1789760804.988195:0:5431:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872c000 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:02bc4b43-5e6f-4d7e-9224-80f5c789104f:5431:1876700024550528:192.168.203.107@tcp:501:ll_cfg_requeue.0 00000100:00100000:3.0:1789760804.988240:0:5431:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c0900b000 going to sleep for 11 seconds 00000100:00100000:3.0:1789760804.988933:0:5431:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872c000 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:02bc4b43-5e6f-4d7e-9224-80f5c789104f:5431:1876700024550528:192.168.203.107@tcp:501:ll_cfg_requeue.0 00000100:00100000:3.0:1789760804.988964:0:5431:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872c000 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:02bc4b43-5e6f-4d7e-9224-80f5c789104f:5431:1876700024550656:192.168.203.107@tcp:503:ll_cfg_requeue.0 00000100:00100000:3.0:1789760804.989055:0:5431:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c0900b000 going to sleep for 11 seconds 00000100:00100000:3.0:1789760804.989686:0:5431:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872c000 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:02bc4b43-5e6f-4d7e-9224-80f5c789104f:5431:1876700024550656:192.168.203.107@tcp:503:ll_cfg_requeue.0 00000040:00080000:1.0:1789760804.990225:0:9472:0:(llog.c:862:llog_process_thread()) stop processing plain [0xa:0x2:0x0] index 64768 count 7 00000020:01000000:3.0:1789760804.990274:0:5431:0:(obd_config.c:2145:class_config_parse_llog()) Processed log params gen 7-6 (rc=0) 10000000:01000000:3.0:1789760804.990291:0:5431:0:(mgc_request.c:1879:mgc_process_log()) MGC192.168.203.107@tcp: configuration from log 'params' succeeded (0). 00010000:00010000:3.0:1789760804.990296:0:5431:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.203.107@tcp lock: ffffa06c110ca800/0x59bc99bb61948138 lrc: 3/1,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 3 type: PLN flags: 0x11000000000000 nid: local remote: 0x1daf3002fe9e0986 expref: -99 pid: 5431 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1789760804.990304:0:5431:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: MGC192.168.203.107@tcp lock: ffffa06c110ca800/0x59bc99bb61948138 lrc: 2/0,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 3 type: PLN flags: 0x11000000000000 nid: local remote: 0x1daf3002fe9e0986 expref: -99 pid: 5431 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1789760805.239041:0:2342:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2342:1876700024550784:192.168.203.107@tcp:400:kworker.0 00000100:00100000:0.0:1789760805.239144:0:2342:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:b837c2fb-d1c1-4622-807e-a243f6fc2450:2342:1876700024551040:192.168.203.107@tcp:400:kworker.0 00000100:00100000:0.0:1789760805.239178:0:2342:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:b837c2fb-d1c1-4622-807e-a243f6fc2450:2342:1876700024551296:192.168.203.107@tcp:400:kworker.0 00000100:00100000:0.0:1789760805.239217:0:2342:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [0->1] 00000100:00100000:0.0:1789760805.239223:0:2342:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2342:1876700024550912:192.168.203.107@tcp:400:kworker.0 00000100:00100000:0.0:1789760805.239248:0:2342:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:b837c2fb-d1c1-4622-807e-a243f6fc2450:2342:1876700024551168:192.168.203.107@tcp:400:kworker.0 00000100:00100000:0.0:1789760805.240425:0:2342:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2342:1876700024550784:192.168.203.107@tcp:400:kworker.0 00000100:00100000:0.0:1789760805.240445:0:2342:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:b837c2fb-d1c1-4622-807e-a243f6fc2450:2342:1876700024551040:192.168.203.107@tcp:400:kworker.0 00000100:00100000:0.0:1789760805.240483:0:2342:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2342:1876700024550912:192.168.203.107@tcp:400:kworker.0 00010000:00080000:0.0:1789760805.240520:0:2342:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61549) req@ffffa06c091d2d80 x1876700024551296/t0(0) o400->lustre-OST0001-osc-ffffa06c07e46000@192.168.203.107@tcp:28/4 lens 224/224 e 0 to 0 dl 1789760821 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760805.240532:0:2342:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:b837c2fb-d1c1-4622-807e-a243f6fc2450:2342:1876700024551296:192.168.203.107@tcp:400:kworker.0 00010000:00080000:0.0:1789760805.240558:0:2342:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61549) req@ffffa06c091d0700 x1876700024551168/t0(0) o400->lustre-OST0000-osc-ffffa06c07e46000@192.168.203.107@tcp:28/4 lens 224/224 e 0 to 0 dl 1789760821 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760805.240569:0:2342:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:b837c2fb-d1c1-4622-807e-a243f6fc2450:2342:1876700024551168:192.168.203.107@tcp:400:kworker.0 00010000:00010000:2.0:1789760805.494939:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789760806.519041:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000001:02000400:3.0:1789760806.551147:0:9910:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: === sanityn: finish setup 15:46:46 (1789760806) === 00000080:00200000:2.0:1789760806.578803:0:7453:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908),name=/ 00000002:00010000:2.0:1789760806.578833:0:7453:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1789760806.578873:0:7453:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c19d4c000/0x59bc99bb619480ba lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1daf3002fe9e0898 expref: -99 pid: 5456 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 1 00010000:00010000:2.0:1789760806.578886:0:7453:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c19d4c000/0x59bc99bb619480ba lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1daf3002fe9e0898 expref: -99 pid: 5456 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 1 00000080:00200000:2.0:1789760806.578896:0:7453:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1789760806.578905:0:7453:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa06c03518908) 00000080:00010000:2.0:1789760806.578909:0:7453:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa06c03518908) for lock 0x59bc99bb619480ba 00000080:00010000:2.0:1789760806.578917:0:7453:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x59bc99bb619480ba from it ffffafafc1c3bbb0 00010000:00010000:2.0:1789760806.578922:0:7453:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c19d4c000/0x59bc99bb619480ba lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1daf3002fe9e0898 expref: -99 pid: 5456 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 1 00010000:00010000:2.0:1789760806.578934:0:7453:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c19d4c000/0x59bc99bb619480ba lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1daf3002fe9e0898 expref: -99 pid: 5456 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 1 00000080:00200000:2.0:1789760806.578952:0:7453:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908), inode mode 41ed mask 44 00000080:00200008:2.0:1789760806.578959:0:7453:0:(file.c:1090:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffffa06c03518908), flags 304000 00000080:00200008:2.0:1789760806.578969:0:7453:0:(file.c:6296:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffffa06c03518908), request_mask 2047, flags 0, foreign 0 00000002:00010000:2.0:1789760806.578981:0:7453:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: readdir flags 0 00010000:00010000:2.0:1789760806.578988:0:7453:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c19d4c000/0x59bc99bb619480ba lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1daf3002fe9e0898 expref: -99 pid: 5456 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 1 00010000:00010000:2.0:1789760806.578997:0:7453:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c19d4c000/0x59bc99bb619480ba lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1daf3002fe9e0898 expref: -99 pid: 5456 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 1 00000100:00100000:2.0:1789760806.579515:0:7453:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e52a00 pname:cluuid:pid:xid:nid:opc:job bash:734aaee5-e21a-40f9-9d4e-90e3c76738d2:7453:1876700024551424:192.168.203.107@tcp:37:bash.0 00000100:00100000:2.0:1789760806.579627:0:7453:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c04e32300 going to sleep for 11 seconds 00000100:00100000:0.0:1789760806.583016:0:7453:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e52a00 pname:cluuid:pid:xid:nid:opc:job bash:734aaee5-e21a-40f9-9d4e-90e3c76738d2:7453:1876700024551424:192.168.203.107@tcp:37:bash.0 00010000:00010000:0.0:1789760806.583657:0:7453:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c19d4c000/0x59bc99bb619480ba lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1daf3002fe9e0898 expref: -99 pid: 5456 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 1 00010000:00010000:0.0:1789760806.583671:0:7453:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c19d4c000/0x59bc99bb619480ba lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1daf3002fe9e0898 expref: -99 pid: 5456 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 1 00000080:00200000:0.0:1789760806.583699:0:7453:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908),name=/ 00000002:00010000:0.0:1789760806.583709:0:7453:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1789760806.583717:0:7453:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c19d4c000/0x59bc99bb619480ba lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1daf3002fe9e0898 expref: -99 pid: 5456 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 1 00010000:00010000:0.0:1789760806.583725:0:7453:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c19d4c000/0x59bc99bb619480ba lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1daf3002fe9e0898 expref: -99 pid: 5456 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 1 00000080:00200000:0.0:1789760806.583729:0:7453:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1789760806.583734:0:7453:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa06c03518908) 00000080:00010000:0.0:1789760806.583736:0:7453:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa06c03518908) for lock 0x59bc99bb619480ba 00000080:00010000:0.0:1789760806.583739:0:7453:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x59bc99bb619480ba from it ffffafafc1c3bcd8 00010000:00010000:0.0:1789760806.583741:0:7453:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c19d4c000/0x59bc99bb619480ba lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1daf3002fe9e0898 expref: -99 pid: 5456 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 1 00010000:00010000:0.0:1789760806.583747:0:7453:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c19d4c000/0x59bc99bb619480ba lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1daf3002fe9e0898 expref: -99 pid: 5456 timeout: 0 lvb_type: 0 lru_score: 11 lru_type: 1 00000080:00200000:0.0:1789760806.584052:0:7453:0:(dir.c:332:ll_iterate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908) pos/size0/4096 32bit_api 0 00000080:00200000:0.0:1789760806.584076:0:7453:0:(dir.c:332:ll_iterate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908) pos/size18446744073709551614/4096 32bit_api 0 00000080:00200008:0.0:1789760806.584101:0:7453:0:(file.c:479:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffffa06c03518908), flags 304000 00000080:00200000:0.0:1789760806.584130:0:7453:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908),name=/ 00000002:00010000:0.0:1789760806.584135:0:7453:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1789760806.584141:0:7453:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c19d4c000/0x59bc99bb619480ba lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1daf3002fe9e0898 expref: -99 pid: 5456 timeout: 0 lvb_type: 0 lru_score: 11 lru_type: 1 00010000:00010000:0.0:1789760806.584150:0:7453:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c19d4c000/0x59bc99bb619480ba lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1daf3002fe9e0898 expref: -99 pid: 5456 timeout: 0 lvb_type: 0 lru_score: 11 lru_type: 1 00000080:00200000:0.0:1789760806.584162:0:7453:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1789760806.584167:0:7453:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa06c03518908) 00000080:00010000:0.0:1789760806.584169:0:7453:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa06c03518908) for lock 0x59bc99bb619480ba 00000080:00010000:0.0:1789760806.584174:0:7453:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x59bc99bb619480ba from it ffffafafc1c3bbb0 00010000:00010000:0.0:1789760806.584178:0:7453:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c19d4c000/0x59bc99bb619480ba lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1daf3002fe9e0898 expref: -99 pid: 5456 timeout: 0 lvb_type: 0 lru_score: 11 lru_type: 1 00010000:00010000:0.0:1789760806.584186:0:7453:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c19d4c000/0x59bc99bb619480ba lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1daf3002fe9e0898 expref: -99 pid: 5456 timeout: 0 lvb_type: 0 lru_score: 12 lru_type: 1 00000080:00200000:0.0:1789760806.584195:0:7453:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908), inode mode 41ed mask 44 00000080:00200008:0.0:1789760806.584201:0:7453:0:(file.c:1090:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffffa06c03518908), flags 304000 00000080:00200008:0.0:1789760806.584213:0:7453:0:(file.c:6296:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffffa06c03518908), request_mask 2047, flags 0, foreign 0 00000080:00200000:0.0:1789760806.584225:0:7453:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908),name=/ 00000002:00010000:0.0:1789760806.584231:0:7453:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1789760806.584239:0:7453:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c19d4c000/0x59bc99bb619480ba lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1daf3002fe9e0898 expref: -99 pid: 5456 timeout: 0 lvb_type: 0 lru_score: 12 lru_type: 1 00010000:00010000:0.0:1789760806.584248:0:7453:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c19d4c000/0x59bc99bb619480ba lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1daf3002fe9e0898 expref: -99 pid: 5456 timeout: 0 lvb_type: 0 lru_score: 12 lru_type: 1 00000080:00200000:0.0:1789760806.584254:0:7453:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1789760806.584258:0:7453:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa06c03518908) 00000080:00010000:0.0:1789760806.584261:0:7453:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa06c03518908) for lock 0x59bc99bb619480ba 00000080:00010000:0.0:1789760806.584265:0:7453:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x59bc99bb619480ba from it ffffafafc1c3bcd8 00010000:00010000:0.0:1789760806.584268:0:7453:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c19d4c000/0x59bc99bb619480ba lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1daf3002fe9e0898 expref: -99 pid: 5456 timeout: 0 lvb_type: 0 lru_score: 12 lru_type: 1 00010000:00010000:0.0:1789760806.584282:0:7453:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c19d4c000/0x59bc99bb619480ba lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1daf3002fe9e0898 expref: -99 pid: 5456 timeout: 0 lvb_type: 0 lru_score: 13 lru_type: 1 00000080:00200000:0.0:1789760806.584322:0:7453:0:(dir.c:332:ll_iterate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908) pos/size0/4096 32bit_api 0 00000080:00200000:0.0:1789760806.584339:0:7453:0:(dir.c:332:ll_iterate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908) pos/size18446744073709551614/4096 32bit_api 0 00000080:00200008:0.0:1789760806.584347:0:7453:0:(file.c:479:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffffa06c03518908), flags 304000 00000080:00200000:1.0:1789760806.588441:0:9919:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908),name=/ 00000002:00010000:1.0:1789760806.588467:0:9919:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1789760806.588488:0:9919:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c19d4c000/0x59bc99bb619480ba lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1daf3002fe9e0898 expref: -99 pid: 5456 timeout: 0 lvb_type: 0 lru_score: 13 lru_type: 1 00010000:00010000:1.0:1789760806.588505:0:9919:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c19d4c000/0x59bc99bb619480ba lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1daf3002fe9e0898 expref: -99 pid: 5456 timeout: 0 lvb_type: 0 lru_score: 13 lru_type: 1 00000080:00200000:1.0:1789760806.588514:0:9919:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1789760806.588522:0:9919:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa06c03518908) 00000080:00010000:1.0:1789760806.588526:0:9919:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa06c03518908) for lock 0x59bc99bb619480ba 00000080:00010000:1.0:1789760806.588534:0:9919:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x59bc99bb619480ba from it ffffafafc1d23b58 00010000:00010000:1.0:1789760806.588538:0:9919:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c19d4c000/0x59bc99bb619480ba lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1daf3002fe9e0898 expref: -99 pid: 5456 timeout: 0 lvb_type: 0 lru_score: 13 lru_type: 1 00010000:00010000:1.0:1789760806.588548:0:9919:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c19d4c000/0x59bc99bb619480ba lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1daf3002fe9e0898 expref: -99 pid: 5456 timeout: 0 lvb_type: 0 lru_score: 14 lru_type: 1 00000080:00200000:1.0:1789760806.588562:0:9919:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908), inode mode 41ed mask 1 00000080:00200000:1.0:1789760806.588575:0:9919:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=[df][0-9]*, dir=[0x200000007:0x1:0x0](ffffa06c03518908), flags=8 00000080:00200000:1.0:1789760806.588581:0:9919:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=[df][0-9]*, dir=[0x200000007:0x1:0x0](ffffa06c03518908), intent=getattr 00000002:00010000:1.0:1789760806.588596:0:9919:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: [df][0-9]*,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 01 00010000:00010000:1.0:1789760806.588666:0:9919:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c19d4c600/0x59bc99bb6194813f lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 9919 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1789760806.588676:0:9919:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c19d4c600/0x59bc99bb6194813f lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 9919 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1789760806.588687:0:9919:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c19d4c600/0x59bc99bb6194813f lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 9919 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1789760806.588723:0:9919:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2ca80 pname:cluuid:pid:xid:nid:opc:job rm:734aaee5-e21a-40f9-9d4e-90e3c76738d2:9919:1876700024551552:192.168.203.107@tcp:101:rm.0 00000100:00100000:1.0:1789760806.588843:0:9919:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c10a750c0 going to sleep for 11 seconds 00000100:00100000:2.0:1789760806.592406:0:9919:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2ca80 pname:cluuid:pid:xid:nid:opc:job rm:734aaee5-e21a-40f9-9d4e-90e3c76738d2:9919:1876700024551552:192.168.203.107@tcp:101:rm.0 00010000:00010000:2.0:1789760806.592426:0:9919:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c19d4c600/0x59bc99bb6194813f lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 9919 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1789760806.592439:0:9919:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c19d4c600/0x59bc99bb6194813f lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 9919 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1789760806.592447:0:9919:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c19d4c600/0x59bc99bb6194813f lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 9919 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1789760806.592455:0:9919:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c19d4c600/0x59bc99bb6194813f lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 9919 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1789760806.592463:0:9919:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c19d4c600/0x59bc99bb6194813f lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 9919 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1789760806.592470:0:9919:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffffa06c19d4c600 already unused, calling callback (ffffffffc1b7f460) 00010000:00010000:2.0:1789760806.592474:0:9919:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c19d4c600/0x59bc99bb6194813f lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 9919 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1789760806.592481:0:9919:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c19d4c600/0x59bc99bb6194813f lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 9919 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1789760806.592488:0:9919:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c19d4c600/0x59bc99bb6194813f lrc: 5/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x20e400000000 pid: 9919 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1789760806.592497:0:9919:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:2.0:1789760806.592505:0:9919:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c19d4c600/0x59bc99bb6194813f lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 9919 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1789760806.592522:0:9919:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c19d4c600/0x59bc99bb6194813f lrc: 0/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 9919 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1789760806.592542:0:9919:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0x7, status=0 req@ffffa06c08e2ca80 x1876700024551552/t0(0) o101->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 584/584 e 0 to 0 dl 1789760822 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'rm.0' uid:0 gid:0 projid:0 00000080:00200000:2.0:1789760806.592555:0:9919:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc 0 00010000:00010000:2.0:1789760806.592576:0:9919:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c19d4c000/0x59bc99bb619480ba lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1daf3002fe9e0898 expref: -99 pid: 5456 timeout: 0 lvb_type: 0 lru_score: 14 lru_type: 1 00010000:00010000:2.0:1789760806.592587:0:9919:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c19d4c000/0x59bc99bb619480ba lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1daf3002fe9e0898 expref: -99 pid: 5456 timeout: 0 lvb_type: 0 lru_score: 14 lru_type: 1 00000080:00010000:2.0:1789760806.592595:0:9919:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x59bc99bb619480ba from it ffffafafc1d23930 00010000:00010000:2.0:1789760806.592599:0:9919:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c19d4c000/0x59bc99bb619480ba lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1daf3002fe9e0898 expref: -99 pid: 5456 timeout: 0 lvb_type: 0 lru_score: 14 lru_type: 1 00010000:00010000:2.0:1789760806.592606:0:9919:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c19d4c000/0x59bc99bb619480ba lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1daf3002fe9e0898 expref: -99 pid: 5456 timeout: 0 lvb_type: 0 lru_score: 15 lru_type: 1 00000080:00200000:2.0:1789760806.592675:0:9919:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908),name=/ 00000002:00010000:2.0:1789760806.592688:0:9919:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1789760806.592697:0:9919:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c19d4c000/0x59bc99bb619480ba lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1daf3002fe9e0898 expref: -99 pid: 5456 timeout: 0 lvb_type: 0 lru_score: 15 lru_type: 1 00010000:00010000:2.0:1789760806.592706:0:9919:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c19d4c000/0x59bc99bb619480ba lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1daf3002fe9e0898 expref: -99 pid: 5456 timeout: 0 lvb_type: 0 lru_score: 15 lru_type: 1 00000080:00200000:2.0:1789760806.592712:0:9919:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1789760806.592724:0:9919:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa06c03518908) 00000080:00010000:2.0:1789760806.592728:0:9919:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa06c03518908) for lock 0x59bc99bb619480ba 00000080:00010000:2.0:1789760806.592733:0:9919:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x59bc99bb619480ba from it ffffafafc1d23c10 00010000:00010000:2.0:1789760806.592736:0:9919:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c19d4c000/0x59bc99bb619480ba lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1daf3002fe9e0898 expref: -99 pid: 5456 timeout: 0 lvb_type: 0 lru_score: 15 lru_type: 1 00010000:00010000:2.0:1789760806.592743:0:9919:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c19d4c000/0x59bc99bb619480ba lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1daf3002fe9e0898 expref: -99 pid: 5456 timeout: 0 lvb_type: 0 lru_score: 16 lru_type: 1 00000080:00200000:2.0:1789760806.592752:0:9919:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908), inode mode 41ed mask 1 00000080:00200000:2.0:1789760806.592760:0:9919:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=[df][0-9]*, flags=0 00000080:00200000:2.0:1789760806.592777:0:9919:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908),name=/ 00000002:00010000:2.0:1789760806.592785:0:9919:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1789760806.592792:0:9919:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c19d4c000/0x59bc99bb619480ba lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1daf3002fe9e0898 expref: -99 pid: 5456 timeout: 0 lvb_type: 0 lru_score: 16 lru_type: 1 00010000:00010000:2.0:1789760806.592801:0:9919:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c19d4c000/0x59bc99bb619480ba lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1daf3002fe9e0898 expref: -99 pid: 5456 timeout: 0 lvb_type: 0 lru_score: 16 lru_type: 1 00000080:00200000:2.0:1789760806.592807:0:9919:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1789760806.592812:0:9919:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa06c03518908) 00000080:00010000:2.0:1789760806.592815:0:9919:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa06c03518908) for lock 0x59bc99bb619480ba 00000080:00010000:2.0:1789760806.592819:0:9919:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x59bc99bb619480ba from it ffffafafc1d23b58 00010000:00010000:2.0:1789760806.592822:0:9919:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c19d4c000/0x59bc99bb619480ba lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1daf3002fe9e0898 expref: -99 pid: 5456 timeout: 0 lvb_type: 0 lru_score: 16 lru_type: 1 00010000:00010000:2.0:1789760806.592830:0:9919:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c19d4c000/0x59bc99bb619480ba lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1daf3002fe9e0898 expref: -99 pid: 5456 timeout: 0 lvb_type: 0 lru_score: 17 lru_type: 1 00000080:00200000:2.0:1789760806.593143:0:9919:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908), inode mode 41ed mask 1 00000080:00200000:2.0:1789760806.593238:0:9919:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=lnk, dir=[0x200000007:0x1:0x0](ffffa06c03518908), flags=8 00000080:00200000:2.0:1789760806.593243:0:9919:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=lnk, dir=[0x200000007:0x1:0x0](ffffa06c03518908), intent=getattr 00000002:00010000:2.0:1789760806.593256:0:9919:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: lnk,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 01 00010000:00010000:2.0:1789760806.593289:0:9919:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c01513800/0x59bc99bb61948146 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 9919 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1789760806.593295:0:9919:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c01513800/0x59bc99bb61948146 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 9919 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1789760806.593303:0:9919:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c01513800/0x59bc99bb61948146 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 9919 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1789760806.593321:0:9919:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50700 pname:cluuid:pid:xid:nid:opc:job rm:734aaee5-e21a-40f9-9d4e-90e3c76738d2:9919:1876700024551680:192.168.203.107@tcp:101:rm.0 00000100:00100000:2.0:1789760806.593387:0:9919:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c04e32d80 going to sleep for 11 seconds 00000100:00100000:2.0:1789760806.594612:0:9919:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50700 pname:cluuid:pid:xid:nid:opc:job rm:734aaee5-e21a-40f9-9d4e-90e3c76738d2:9919:1876700024551680:192.168.203.107@tcp:101:rm.0 00010000:00010000:2.0:1789760806.594622:0:9919:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c01513800/0x59bc99bb61948146 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 9919 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1789760806.594630:0:9919:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c01513800/0x59bc99bb61948146 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 9919 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1789760806.594646:0:9919:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c01513800/0x59bc99bb61948146 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 9919 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1789760806.594653:0:9919:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c01513800/0x59bc99bb61948146 lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 9919 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1789760806.594671:0:9919:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c01513800/0x59bc99bb61948146 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 9919 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1789760806.594677:0:9919:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffffa06c01513800 already unused, calling callback (ffffffffc1b7f460) 00010000:00010000:2.0:1789760806.594681:0:9919:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c01513800/0x59bc99bb61948146 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 9919 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1789760806.594687:0:9919:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c01513800/0x59bc99bb61948146 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 9919 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1789760806.594693:0:9919:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c01513800/0x59bc99bb61948146 lrc: 5/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x20e400000000 pid: 9919 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1789760806.594699:0:9919:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:2.0:1789760806.594704:0:9919:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c01513800/0x59bc99bb61948146 lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 9919 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1789760806.594710:0:9919:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c01513800/0x59bc99bb61948146 lrc: 0/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 9919 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1789760806.594726:0:9919:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0x7, status=0 req@ffffa06c07e50700 x1876700024551680/t0(0) o101->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 576/584 e 0 to 0 dl 1789760822 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'rm.0' uid:0 gid:0 projid:0 00000080:00200000:2.0:1789760806.594734:0:9919:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc 0 00010000:00010000:2.0:1789760806.594754:0:9919:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c19d4c000/0x59bc99bb619480ba lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1daf3002fe9e0898 expref: -99 pid: 5456 timeout: 0 lvb_type: 0 lru_score: 17 lru_type: 1 00010000:00010000:2.0:1789760806.594760:0:9919:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c19d4c000/0x59bc99bb619480ba lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1daf3002fe9e0898 expref: -99 pid: 5456 timeout: 0 lvb_type: 0 lru_score: 17 lru_type: 1 00000080:00010000:2.0:1789760806.594765:0:9919:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x59bc99bb619480ba from it ffffafafc1d23930 00010000:00010000:2.0:1789760806.594772:0:9919:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c19d4c000/0x59bc99bb619480ba lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1daf3002fe9e0898 expref: -99 pid: 5456 timeout: 0 lvb_type: 0 lru_score: 17 lru_type: 1 00010000:00010000:2.0:1789760806.594777:0:9919:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c19d4c000/0x59bc99bb619480ba lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1daf3002fe9e0898 expref: -99 pid: 5456 timeout: 0 lvb_type: 0 lru_score: 18 lru_type: 1 00000080:00200000:2.0:1789760806.594803:0:9919:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908),name=/ 00000002:00010000:2.0:1789760806.594809:0:9919:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1789760806.594814:0:9919:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c19d4c000/0x59bc99bb619480ba lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1daf3002fe9e0898 expref: -99 pid: 5456 timeout: 0 lvb_type: 0 lru_score: 18 lru_type: 1 00010000:00010000:2.0:1789760806.594820:0:9919:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c19d4c000/0x59bc99bb619480ba lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1daf3002fe9e0898 expref: -99 pid: 5456 timeout: 0 lvb_type: 0 lru_score: 18 lru_type: 1 00000080:00200000:2.0:1789760806.594824:0:9919:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1789760806.594826:0:9919:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa06c03518908) 00000080:00010000:2.0:1789760806.594828:0:9919:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa06c03518908) for lock 0x59bc99bb619480ba 00000080:00010000:2.0:1789760806.594830:0:9919:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x59bc99bb619480ba from it ffffafafc1d23c10 00010000:00010000:2.0:1789760806.594832:0:9919:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c19d4c000/0x59bc99bb619480ba lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1daf3002fe9e0898 expref: -99 pid: 5456 timeout: 0 lvb_type: 0 lru_score: 18 lru_type: 1 00010000:00010000:2.0:1789760806.594837:0:9919:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c19d4c000/0x59bc99bb619480ba lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1daf3002fe9e0898 expref: -99 pid: 5456 timeout: 0 lvb_type: 0 lru_score: 19 lru_type: 1 00000080:00200000:2.0:1789760806.594842:0:9919:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908), inode mode 41ed mask 1 00000080:00200000:2.0:1789760806.594865:0:9919:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=lnk, flags=0 00000080:00200000:2.0:1789760806.594877:0:9919:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908),name=/ 00000002:00010000:2.0:1789760806.594882:0:9919:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1789760806.594890:0:9919:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c19d4c000/0x59bc99bb619480ba lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1daf3002fe9e0898 expref: -99 pid: 5456 timeout: 0 lvb_type: 0 lru_score: 19 lru_type: 1 00010000:00010000:2.0:1789760806.594901:0:9919:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c19d4c000/0x59bc99bb619480ba lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1daf3002fe9e0898 expref: -99 pid: 5456 timeout: 0 lvb_type: 0 lru_score: 19 lru_type: 1 00000080:00200000:2.0:1789760806.594905:0:9919:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1789760806.594909:0:9919:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa06c03518908) 00000080:00010000:2.0:1789760806.594911:0:9919:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa06c03518908) for lock 0x59bc99bb619480ba 00000080:00010000:2.0:1789760806.594916:0:9919:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x59bc99bb619480ba from it ffffafafc1d23b58 00010000:00010000:2.0:1789760806.594920:0:9919:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c19d4c000/0x59bc99bb619480ba lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1daf3002fe9e0898 expref: -99 pid: 5456 timeout: 0 lvb_type: 0 lru_score: 19 lru_type: 1 00010000:00010000:2.0:1789760806.594925:0:9919:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c19d4c000/0x59bc99bb619480ba lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1daf3002fe9e0898 expref: -99 pid: 5456 timeout: 0 lvb_type: 0 lru_score: 20 lru_type: 1 00000080:00200000:2.0:1789760806.594930:0:9919:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908), inode mode 41ed mask 1 00000080:00200000:2.0:1789760806.594936:0:9919:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=[df].sanityn*, dir=[0x200000007:0x1:0x0](ffffa06c03518908), flags=8 00000080:00200000:2.0:1789760806.594938:0:9919:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=[df].sanityn*, dir=[0x200000007:0x1:0x0](ffffa06c03518908), intent=getattr 00000002:00010000:2.0:1789760806.594951:0:9919:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: [df].sanityn*,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 01 00010000:00010000:2.0:1789760806.594967:0:9919:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c01513400/0x59bc99bb6194814d lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 9919 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1789760806.594972:0:9919:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c01513400/0x59bc99bb6194814d lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 9919 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1789760806.594977:0:9919:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c01513400/0x59bc99bb6194814d lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 9919 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1789760806.594985:0:9919:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50700 pname:cluuid:pid:xid:nid:opc:job rm:734aaee5-e21a-40f9-9d4e-90e3c76738d2:9919:1876700024551808:192.168.203.107@tcp:101:rm.0 00000100:00100000:2.0:1789760806.595018:0:9919:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c04e32d80 going to sleep for 11 seconds 00000100:00100000:2.0:1789760806.598215:0:9919:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50700 pname:cluuid:pid:xid:nid:opc:job rm:734aaee5-e21a-40f9-9d4e-90e3c76738d2:9919:1876700024551808:192.168.203.107@tcp:101:rm.0 00010000:00010000:2.0:1789760806.598227:0:9919:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c01513400/0x59bc99bb6194814d lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 9919 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1789760806.598235:0:9919:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c01513400/0x59bc99bb6194814d lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 9919 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1789760806.598240:0:9919:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c01513400/0x59bc99bb6194814d lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 9919 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1789760806.598244:0:9919:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c01513400/0x59bc99bb6194814d lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 9919 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1789760806.598249:0:9919:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c01513400/0x59bc99bb6194814d lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 9919 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1789760806.598253:0:9919:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffffa06c01513400 already unused, calling callback (ffffffffc1b7f460) 00010000:00010000:2.0:1789760806.598255:0:9919:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c01513400/0x59bc99bb6194814d lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 9919 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1789760806.598259:0:9919:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c01513400/0x59bc99bb6194814d lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 9919 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1789760806.598263:0:9919:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c01513400/0x59bc99bb6194814d lrc: 5/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x20e400000000 pid: 9919 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1789760806.598273:0:9919:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:2.0:1789760806.598277:0:9919:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c01513400/0x59bc99bb6194814d lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 9919 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1789760806.598290:0:9919:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c01513400/0x59bc99bb6194814d lrc: 0/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 9919 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1789760806.598301:0:9919:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0x7, status=0 req@ffffa06c07e50700 x1876700024551808/t0(0) o101->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 584/584 e 0 to 0 dl 1789760822 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'rm.0' uid:0 gid:0 projid:0 00000080:00200000:2.0:1789760806.598309:0:9919:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc 0 00010000:00010000:2.0:1789760806.598324:0:9919:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c19d4c000/0x59bc99bb619480ba lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1daf3002fe9e0898 expref: -99 pid: 5456 timeout: 0 lvb_type: 0 lru_score: 20 lru_type: 1 00010000:00010000:2.0:1789760806.598331:0:9919:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c19d4c000/0x59bc99bb619480ba lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1daf3002fe9e0898 expref: -99 pid: 5456 timeout: 0 lvb_type: 0 lru_score: 20 lru_type: 1 00000080:00010000:2.0:1789760806.598336:0:9919:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x59bc99bb619480ba from it ffffafafc1d23930 00010000:00010000:2.0:1789760806.598338:0:9919:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c19d4c000/0x59bc99bb619480ba lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1daf3002fe9e0898 expref: -99 pid: 5456 timeout: 0 lvb_type: 0 lru_score: 20 lru_type: 1 00010000:00010000:2.0:1789760806.598343:0:9919:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c19d4c000/0x59bc99bb619480ba lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1daf3002fe9e0898 expref: -99 pid: 5456 timeout: 0 lvb_type: 0 lru_score: 21 lru_type: 1 00000080:00200000:2.0:1789760806.598387:0:9919:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908),name=/ 00000002:00010000:2.0:1789760806.598400:0:9919:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1789760806.598408:0:9919:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c19d4c000/0x59bc99bb619480ba lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1daf3002fe9e0898 expref: -99 pid: 5456 timeout: 0 lvb_type: 0 lru_score: 21 lru_type: 1 00010000:00010000:2.0:1789760806.598415:0:9919:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c19d4c000/0x59bc99bb619480ba lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1daf3002fe9e0898 expref: -99 pid: 5456 timeout: 0 lvb_type: 0 lru_score: 21 lru_type: 1 00000080:00200000:2.0:1789760806.598419:0:9919:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1789760806.598429:0:9919:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa06c03518908) 00000080:00010000:2.0:1789760806.598432:0:9919:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa06c03518908) for lock 0x59bc99bb619480ba 00000080:00010000:2.0:1789760806.598435:0:9919:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x59bc99bb619480ba from it ffffafafc1d23c10 00010000:00010000:2.0:1789760806.598437:0:9919:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c19d4c000/0x59bc99bb619480ba lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1daf3002fe9e0898 expref: -99 pid: 5456 timeout: 0 lvb_type: 0 lru_score: 21 lru_type: 1 00010000:00010000:2.0:1789760806.598442:0:9919:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c19d4c000/0x59bc99bb619480ba lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1daf3002fe9e0898 expref: -99 pid: 5456 timeout: 0 lvb_type: 0 lru_score: 22 lru_type: 1 00000080:00200000:2.0:1789760806.598448:0:9919:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908), inode mode 41ed mask 1 00000080:00200000:2.0:1789760806.598454:0:9919:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=[df].sanityn*, flags=0 00000080:00200000:0.0:1789760806.649326:0:9929:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908),name=/ 00000002:00010000:0.0:1789760806.649346:0:9929:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1789760806.649360:0:9929:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c19d4c000/0x59bc99bb619480ba lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1daf3002fe9e0898 expref: -99 pid: 5456 timeout: 0 lvb_type: 0 lru_score: 22 lru_type: 1 00010000:00010000:0.0:1789760806.649372:0:9929:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c19d4c000/0x59bc99bb619480ba lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1daf3002fe9e0898 expref: -99 pid: 5456 timeout: 0 lvb_type: 0 lru_score: 22 lru_type: 1 00000080:00200000:0.0:1789760806.649378:0:9929:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1789760806.649384:0:9929:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa06c03518908) 00000080:00010000:0.0:1789760806.649386:0:9929:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa06c03518908) for lock 0x59bc99bb619480ba 00000080:00010000:0.0:1789760806.649391:0:9929:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x59bc99bb619480ba from it ffffafafc1cbbbe0 00010000:00010000:0.0:1789760806.649394:0:9929:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c19d4c000/0x59bc99bb619480ba lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1daf3002fe9e0898 expref: -99 pid: 5456 timeout: 0 lvb_type: 0 lru_score: 22 lru_type: 1 00010000:00010000:0.0:1789760806.649400:0:9929:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c19d4c000/0x59bc99bb619480ba lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1daf3002fe9e0898 expref: -99 pid: 5456 timeout: 0 lvb_type: 0 lru_score: 23 lru_type: 1 00000080:00200000:0.0:1789760806.649408:0:9929:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908), inode mode 41ed mask 1 00000080:00200000:0.0:1789760806.649416:0:9929:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=d0_runas_test, dir=[0x200000007:0x1:0x0](ffffa06c03518908), flags=1536 00000080:00200000:0.0:1789760806.649422:0:9929:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908),name=/ 00000002:00010000:0.0:1789760806.649426:0:9929:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1789760806.649430:0:9929:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c19d4c000/0x59bc99bb619480ba lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1daf3002fe9e0898 expref: -99 pid: 5456 timeout: 0 lvb_type: 0 lru_score: 23 lru_type: 1 00010000:00010000:0.0:1789760806.649435:0:9929:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c19d4c000/0x59bc99bb619480ba lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1daf3002fe9e0898 expref: -99 pid: 5456 timeout: 0 lvb_type: 0 lru_score: 23 lru_type: 1 00000080:00200000:0.0:1789760806.649439:0:9929:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1789760806.649441:0:9929:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa06c03518908) 00000080:00010000:0.0:1789760806.649443:0:9929:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa06c03518908) for lock 0x59bc99bb619480ba 00000080:00010000:0.0:1789760806.649444:0:9929:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x59bc99bb619480ba from it ffffafafc1cbbce0 00010000:00010000:0.0:1789760806.649446:0:9929:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c19d4c000/0x59bc99bb619480ba lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1daf3002fe9e0898 expref: -99 pid: 5456 timeout: 0 lvb_type: 0 lru_score: 23 lru_type: 1 00010000:00010000:0.0:1789760806.649451:0:9929:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c19d4c000/0x59bc99bb619480ba lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1daf3002fe9e0898 expref: -99 pid: 5456 timeout: 0 lvb_type: 0 lru_score: 24 lru_type: 1 00000080:00200000:0.0:1789760806.649455:0:9929:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908), inode mode 41ed mask 3 00000080:00200000:0.0:1789760806.649458:0:9929:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908),name=/ 00000002:00010000:0.0:1789760806.649460:0:9929:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1789760806.649463:0:9929:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c19d4c000/0x59bc99bb619480ba lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1daf3002fe9e0898 expref: -99 pid: 5456 timeout: 0 lvb_type: 0 lru_score: 24 lru_type: 1 00010000:00010000:0.0:1789760806.649467:0:9929:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c19d4c000/0x59bc99bb619480ba lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1daf3002fe9e0898 expref: -99 pid: 5456 timeout: 0 lvb_type: 0 lru_score: 24 lru_type: 1 00000080:00200000:0.0:1789760806.649471:0:9929:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1789760806.649476:0:9929:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa06c03518908) 00000080:00010000:0.0:1789760806.649478:0:9929:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa06c03518908) for lock 0x59bc99bb619480ba 00000080:00010000:0.0:1789760806.649480:0:9929:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x59bc99bb619480ba from it ffffafafc1cbbdc8 00010000:00010000:0.0:1789760806.649482:0:9929:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c19d4c000/0x59bc99bb619480ba lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1daf3002fe9e0898 expref: -99 pid: 5456 timeout: 0 lvb_type: 0 lru_score: 24 lru_type: 1 00010000:00010000:0.0:1789760806.649486:0:9929:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c19d4c000/0x59bc99bb619480ba lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1daf3002fe9e0898 expref: -99 pid: 5456 timeout: 0 lvb_type: 0 lru_score: 25 lru_type: 1 00000080:00200000:0.0:1789760806.649490:0:9929:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908), inode mode 41ed mask 3 00000080:00200000:0.0:1789760806.649493:0:9929:0:(namei.c:2239:do_mkdir()) VFS Op:name=d0_runas_test, dir=[0x200000007:0x1:0x0](ffffa06c03518908) 00000100:00100000:0.0:1789760806.649539:0:9929:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0700 pname:cluuid:pid:xid:nid:opc:job mkdir:734aaee5-e21a-40f9-9d4e-90e3c76738d2:9929:1876700024551936:192.168.203.107@tcp:700:mkdir.0 00000100:00100000:0.0:1789760806.649618:0:9929:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200ebb40 going to sleep for 11 seconds 00000100:00100000:3.0:1789760806.650917:0:9929:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0700 pname:cluuid:pid:xid:nid:opc:job mkdir:734aaee5-e21a-40f9-9d4e-90e3c76738d2:9929:1876700024551936:192.168.203.107@tcp:700:mkdir.0 00010000:00010000:3.0:1789760806.650973:0:9929:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c19d4c000/0x59bc99bb619480ba lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x1daf3002fe9e0898 expref: -99 pid: 5456 timeout: 0 lvb_type: 0 lru_score: 25 lru_type: 1 00010000:00010000:3.0:1789760806.651099:0:9929:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c19d4c000/0x59bc99bb619480ba lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x1daf3002fe9e0898 expref: -99 pid: 5456 timeout: 0 lvb_type: 0 lru_score: 25 lru_type: 1 00010000:00010000:3.0:1789760806.651108:0:9929:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:3.0:1789760806.651111:0:9929:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c19d4c000/0x59bc99bb619480ba lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x1daf3002fe9e0898 expref: -99 pid: 5456 timeout: 0 lvb_type: 0 lru_score: 25 lru_type: 1 00000020:00100000:3.0:1789760806.651137:0:9929:0:(genops.c:2428:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffffa06c18a1e000: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:3.0:1789760806.651146:0:9929:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffffa06c1872c000 x1876700024552192/t0(0) o36->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 560/576 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789760806.651168:0:9929:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872c000 pname:cluuid:pid:xid:nid:opc:job mkdir:734aaee5-e21a-40f9-9d4e-90e3c76738d2:9929:1876700024552192:192.168.203.107@tcp:36:mkdir.0 00000100:00100000:3.0:1789760806.651235:0:9929:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c04e42240 going to sleep for 11 seconds 00000100:00100000:3.0:1789760806.652906:0:9929:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872c000 pname:cluuid:pid:xid:nid:opc:job mkdir:734aaee5-e21a-40f9-9d4e-90e3c76738d2:9929:1876700024552192:192.168.203.107@tcp:36:mkdir.0 00000080:00200000:3.0:1789760806.653002:0:9929:0:(llite_lib.c:3356:ll_read_inode2()) VFS Op:inode=[0x200000401:0x1:0x0](ffffa06c03589148) 00000080:00200000:3.0:1789760806.653010:0:9929:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200000401:0x1:0x0], updating i_size 4096 00000080:00200008:3.0:1789760806.657328:0:9930:0:(file.c:6296:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffffa06c03518908), request_mask 2047, flags 0, foreign 0 00000080:00200000:3.0:1789760806.657342:0:9930:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908),name=/ 00000002:00010000:3.0:1789760806.657363:0:9930:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1789760806.657421:0:9930:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c42174400/0x59bc99bb61948154 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 9930 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1789760806.657442:0:9930:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c42174400/0x59bc99bb61948154 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 9930 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1789760806.657454:0:9930:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c42174400/0x59bc99bb61948154 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 9930 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1789760806.657477:0:9930:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872d500 pname:cluuid:pid:xid:nid:opc:job chmod:734aaee5-e21a-40f9-9d4e-90e3c76738d2:9930:1876700024552320:192.168.203.107@tcp:101:chmod.0 00000100:00100000:3.0:1789760806.657554:0:9930:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c04e42240 going to sleep for 11 seconds 00000100:00100000:3.0:1789760806.658927:0:9930:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872d500 pname:cluuid:pid:xid:nid:opc:job chmod:734aaee5-e21a-40f9-9d4e-90e3c76738d2:9930:1876700024552320:192.168.203.107@tcp:101:chmod.0 00010000:00010000:3.0:1789760806.658947:0:9930:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c42174400/0x59bc99bb61948154 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1daf3002fe9e0a0b expref: -99 pid: 9930 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1789760806.658959:0:9930:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c42174400/0x59bc99bb61948154 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1daf3002fe9e0a0b expref: -99 pid: 9930 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1789760806.658972:0:9930:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c42174400/0x59bc99bb61948154 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1daf3002fe9e0a0b expref: -99 pid: 9930 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1789760806.658984:0:9930:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c42174400/0x59bc99bb61948154 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1daf3002fe9e0a0b expref: -99 pid: 9930 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1789760806.658996:0:9930:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c42174400/0x59bc99bb61948154 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1daf3002fe9e0a0b expref: -99 pid: 9930 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1789760806.659005:0:9930:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c42174400/0x59bc99bb61948154 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1daf3002fe9e0a0b expref: -99 pid: 9930 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1789760806.659027:0:9930:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c42174400/0x59bc99bb61948154 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1daf3002fe9e0a0b expref: -99 pid: 9930 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:3.0:1789760806.659040:0:9930:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffffa06c1872d500 x1876700024552320/t0(0) o101->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 576/688 e 0 to 0 dl 1789760822 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'chmod.0' uid:0 gid:0 projid:0 00000002:00010000:3.0:1789760806.659055:0:9930:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c42174400/0x59bc99bb61948154 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1daf3002fe9e0a0b expref: -99 pid: 9930 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1789760806.659066:0:9930:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffffa06c18665000 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:3.0:1789760806.659072:0:9930:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:3.0:1789760806.659095:0:9930:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 4096 00000080:00010000:3.0:1789760806.659102:0:9930:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa06c03518908) for lock 0x59bc99bb61948154 00000080:00010000:3.0:1789760806.659107:0:9930:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa06c03518908) 00000080:00010000:3.0:1789760806.659111:0:9930:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x59bc99bb61948154 from it ffffafafc1c63cb8 00010000:00010000:3.0:1789760806.659114:0:9930:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c42174400/0x59bc99bb61948154 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1daf3002fe9e0a0b expref: -99 pid: 9930 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1789760806.659123:0:9930:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c42174400/0x59bc99bb61948154 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1daf3002fe9e0a0b expref: -99 pid: 9930 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200008:3.0:1789760806.659177:0:9930:0:(llite_lib.c:2427:ll_setattr_raw()) START file /:[0x200000007:0x1:0x0](ffffa06c03518908) current size 4096, valid attrs 41, mode 41ed, uid 131305472, gid -24468, new size 0, atime 1789760806.000000000, mtime 1789760806.000000000, ctime 1789760806.000000000 00010000:00010000:3.0:1789760806.659197:0:9930:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c42174400/0x59bc99bb61948154 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x1daf3002fe9e0a0b expref: -99 pid: 9930 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1789760806.659235:0:9930:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c42174400/0x59bc99bb61948154 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x1daf3002fe9e0a0b expref: -99 pid: 9930 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1789760806.659248:0:9930:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:3.0:1789760806.659252:0:9930:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c42174400/0x59bc99bb61948154 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x1daf3002fe9e0a0b expref: -99 pid: 9930 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000020:00100000:3.0:1789760806.659276:0:9930:0:(genops.c:2428:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffffa06c18a1e000: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:3.0:1789760806.659283:0:9930:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffffa06c1872d500 x1876700024552576/t0(0) o36->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 488/512 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789760806.659298:0:9930:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872d500 pname:cluuid:pid:xid:nid:opc:job chmod:734aaee5-e21a-40f9-9d4e-90e3c76738d2:9930:1876700024552576:192.168.203.107@tcp:36:chmod.0 00000100:00100000:3.0:1789760806.659343:0:9930:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c04e42240 going to sleep for 11 seconds 00000100:00100000:3.0:1789760806.660340:0:9930:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872d500 pname:cluuid:pid:xid:nid:opc:job chmod:734aaee5-e21a-40f9-9d4e-90e3c76738d2:9930:1876700024552576:192.168.203.107@tcp:36:chmod.0 00000080:00200000:3.0:1789760806.660361:0:9930:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 4096 00000080:00200008:3.0:1789760806.660369:0:9930:0:(llite_lib.c:2686:ll_setattr_raw()) COMPLETED file /:[0x200000007:0x1:0x0](ffffa06c03518908) current size 4096, valid attrs 10000041, mode 41ed, uid 131305472, gid -24468, new size 0, atime 1789760806.000000000, mtime 1789760806.000000000, ctime 1789760806.000000000, rc 0 00000080:00200000:0.0:1789760806.668032:0:9931:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908),name=/ 00000002:00010000:0.0:1789760806.668050:0:9931:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1789760806.668091:0:9931:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c110cbc00/0x59bc99bb6194815b lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 9931 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1789760806.668099:0:9931:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c110cbc00/0x59bc99bb6194815b lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 9931 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1789760806.668107:0:9931:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c110cbc00/0x59bc99bb6194815b lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 9931 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1789760806.668125:0:9931:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d2d80 pname:cluuid:pid:xid:nid:opc:job chown:734aaee5-e21a-40f9-9d4e-90e3c76738d2:9931:1876700024552704:192.168.203.107@tcp:101:chown.0 00000100:00100000:0.0:1789760806.668197:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 11 seconds 00000100:00100000:0.0:1789760806.669381:0:9931:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d2d80 pname:cluuid:pid:xid:nid:opc:job chown:734aaee5-e21a-40f9-9d4e-90e3c76738d2:9931:1876700024552704:192.168.203.107@tcp:101:chown.0 00010000:00010000:0.0:1789760806.669393:0:9931:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c110cbc00/0x59bc99bb6194815b lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1daf3002fe9e0a20 expref: -99 pid: 9931 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1789760806.669404:0:9931:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c110cbc00/0x59bc99bb6194815b lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1daf3002fe9e0a20 expref: -99 pid: 9931 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1789760806.669412:0:9931:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c110cbc00/0x59bc99bb6194815b lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1daf3002fe9e0a20 expref: -99 pid: 9931 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1789760806.669419:0:9931:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c110cbc00/0x59bc99bb6194815b lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1daf3002fe9e0a20 expref: -99 pid: 9931 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1789760806.669425:0:9931:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c110cbc00/0x59bc99bb6194815b lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1daf3002fe9e0a20 expref: -99 pid: 9931 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1789760806.669431:0:9931:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c110cbc00/0x59bc99bb6194815b lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1daf3002fe9e0a20 expref: -99 pid: 9931 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1789760806.669435:0:9931:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c110cbc00/0x59bc99bb6194815b lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1daf3002fe9e0a20 expref: -99 pid: 9931 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1789760806.669441:0:9931:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffffa06c091d2d80 x1876700024552704/t0(0) o101->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 576/688 e 0 to 0 dl 1789760822 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'chown.0' uid:0 gid:0 projid:0 00000002:00010000:0.0:1789760806.669449:0:9931:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c110cbc00/0x59bc99bb6194815b lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1daf3002fe9e0a20 expref: -99 pid: 9931 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1789760806.669456:0:9931:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffffa06c18665000 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:0.0:1789760806.669459:0:9931:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:0.0:1789760806.669474:0:9931:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 4096 00000080:00010000:0.0:1789760806.669480:0:9931:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa06c03518908) for lock 0x59bc99bb6194815b 00000080:00010000:0.0:1789760806.669484:0:9931:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa06c03518908) 00000080:00010000:0.0:1789760806.669486:0:9931:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x59bc99bb6194815b from it ffffafafc1cb3b58 00010000:00010000:0.0:1789760806.669488:0:9931:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c110cbc00/0x59bc99bb6194815b lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1daf3002fe9e0a20 expref: -99 pid: 9931 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1789760806.669494:0:9931:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c110cbc00/0x59bc99bb6194815b lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1daf3002fe9e0a20 expref: -99 pid: 9931 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1789760806.669509:0:9931:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908), inode mode 41ed mask 1 00000080:00200000:0.0:1789760806.669517:0:9931:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=d0_runas_test, dir=[0x200000007:0x1:0x0](ffffa06c03518908), flags=8 00000080:00200000:0.0:1789760806.669520:0:9931:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=d0_runas_test, dir=[0x200000007:0x1:0x0](ffffa06c03518908), intent=getattr 00000002:00010000:0.0:1789760806.669529:0:9931:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: d0_runas_test,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 01 00010000:00010000:0.0:1789760806.669545:0:9931:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c110cb400/0x59bc99bb61948162 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 9931 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1789760806.669552:0:9931:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c110cb400/0x59bc99bb61948162 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 9931 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1789760806.669558:0:9931:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c110cb400/0x59bc99bb61948162 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 9931 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1789760806.669570:0:9931:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d2d80 pname:cluuid:pid:xid:nid:opc:job chown:734aaee5-e21a-40f9-9d4e-90e3c76738d2:9931:1876700024552832:192.168.203.107@tcp:101:chown.0 00000100:00100000:0.0:1789760806.669607:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 11 seconds 00000100:00100000:0.0:1789760806.670631:0:9931:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d2d80 pname:cluuid:pid:xid:nid:opc:job chown:734aaee5-e21a-40f9-9d4e-90e3c76738d2:9931:1876700024552832:192.168.203.107@tcp:101:chown.0 00010000:00010000:0.0:1789760806.670652:0:9931:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c110cb400/0x59bc99bb61948162 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1daf3002fe9e0a3c expref: -99 pid: 9931 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1789760806.670676:0:9931:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c110cb400/0x59bc99bb61948162 lrc: 4/1,0 mode: --/PR res: [0x200000401:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1daf3002fe9e0a3c expref: -99 pid: 9931 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1789760806.670683:0:9931:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c110cb400/0x59bc99bb61948162 lrc: 4/1,0 mode: --/PR res: [0x200000401:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1daf3002fe9e0a3c expref: -99 pid: 9931 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1789760806.670689:0:9931:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c110cb400/0x59bc99bb61948162 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1daf3002fe9e0a3c expref: -99 pid: 9931 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1789760806.670694:0:9931:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c110cb400/0x59bc99bb61948162 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1daf3002fe9e0a3c expref: -99 pid: 9931 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1789760806.670700:0:9931:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c110cb400/0x59bc99bb61948162 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1daf3002fe9e0a3c expref: -99 pid: 9931 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1789760806.670707:0:9931:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c110cb400/0x59bc99bb61948162 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1daf3002fe9e0a3c expref: -99 pid: 9931 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1789760806.670713:0:9931:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c110cb400/0x59bc99bb61948162 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1daf3002fe9e0a3c expref: -99 pid: 9931 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1789760806.670722:0:9931:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffffa06c091d2d80 x1876700024552832/t0(0) o101->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 584/584 e 0 to 0 dl 1789760822 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'chown.0' uid:0 gid:0 projid:0 00000002:00010000:0.0:1789760806.670732:0:9931:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c110cb400/0x59bc99bb61948162 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1daf3002fe9e0a3c expref: -99 pid: 9931 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1789760806.670741:0:9931:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffffa06c18665000 type 13 mode 4 res 8589935617/1 (0 0) 00000080:00200000:0.0:1789760806.670750:0:9931:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc 0 00000080:00200000:0.0:1789760806.670761:0:9931:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200000401:0x1:0x0], updating i_size 4096 00000080:00200000:0.0:1789760806.670764:0:9931:0:(namei.c:147:ll_iget()) got inode: [0x200000401:0x1:0x0](ffffa06c03589148): rc = 0 00000080:00010000:0.0:1789760806.670768:0:9931:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000401:0x1:0x0](ffffa06c03589148) for lock 0x59bc99bb61948162 00000080:00010000:0.0:1789760806.670774:0:9931:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x1:0x0](ffffa06c03589148) 00000080:00010000:0.0:1789760806.670777:0:9931:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x59bc99bb61948162 from it ffffafafc1cb3b20 00010000:00010000:0.0:1789760806.670780:0:9931:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c110cb400/0x59bc99bb61948162 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1daf3002fe9e0a3c expref: -99 pid: 9931 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1789760806.670788:0:9931:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c110cb400/0x59bc99bb61948162 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1daf3002fe9e0a3c expref: -99 pid: 9931 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200008:0.0:1789760806.670807:0:9931:0:(file.c:6296:ll_getattr_dentry()) START file d0_runas_test:[0x200000401:0x1:0x0](ffffa06c03589148), request_mask 2047, flags 0, foreign 0 00000080:00200000:0.0:1789760806.670811:0:9931:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x1:0x0](ffffa06c03589148),name=d0_runas_test 00000002:00010000:0.0:1789760806.670818:0:9931:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000401:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1789760806.670824:0:9931:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c110cb400/0x59bc99bb61948162 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1daf3002fe9e0a3c expref: -99 pid: 9931 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1789760806.670829:0:9931:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c110cb400/0x59bc99bb61948162 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1daf3002fe9e0a3c expref: -99 pid: 9931 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1789760806.670833:0:9931:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1789760806.670835:0:9931:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x1:0x0](ffffa06c03589148) 00000080:00010000:0.0:1789760806.670837:0:9931:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000401:0x1:0x0](ffffa06c03589148) for lock 0x59bc99bb61948162 00000080:00010000:0.0:1789760806.670839:0:9931:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x59bc99bb61948162 from it ffffafafc1cb3cb8 00010000:00010000:0.0:1789760806.670841:0:9931:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c110cb400/0x59bc99bb61948162 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1daf3002fe9e0a3c expref: -99 pid: 9931 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1789760806.670874:0:9931:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c110cb400/0x59bc99bb61948162 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1daf3002fe9e0a3c expref: -99 pid: 9931 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:0.0:1789760806.670901:0:9931:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908),name=/ 00000002:00010000:0.0:1789760806.670905:0:9931:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1789760806.670910:0:9931:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c110cbc00/0x59bc99bb6194815b lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1daf3002fe9e0a20 expref: -99 pid: 9931 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1789760806.670915:0:9931:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c110cbc00/0x59bc99bb6194815b lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1daf3002fe9e0a20 expref: -99 pid: 9931 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1789760806.670919:0:9931:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1789760806.670921:0:9931:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa06c03518908) 00000080:00010000:0.0:1789760806.670923:0:9931:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa06c03518908) for lock 0x59bc99bb6194815b 00000080:00010000:0.0:1789760806.670925:0:9931:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x59bc99bb6194815b from it ffffafafc1cb3bf8 00010000:00010000:0.0:1789760806.670926:0:9931:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c110cbc00/0x59bc99bb6194815b lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1daf3002fe9e0a20 expref: -99 pid: 9931 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1789760806.670930:0:9931:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c110cbc00/0x59bc99bb6194815b lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1daf3002fe9e0a20 expref: -99 pid: 9931 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:0.0:1789760806.670935:0:9931:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa06c03518908), inode mode 41ed mask 1 00000080:00200000:0.0:1789760806.670937:0:9931:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d0_runas_test, flags=9 00000080:00200008:0.0:1789760806.670947:0:9931:0:(llite_lib.c:2427:ll_setattr_raw()) START file d0_runas_test:[0x200000401:0x1:0x0](ffffa06c03589148) current size 4096, valid attrs 46, mode ffff, uid 500, gid 500, new size 18446638984864575488, atime 1789760806.000000000, mtime 1789760806.000000000, ctime 1789760806.000000000 00010000:00010000:0.0:1789760806.670958:0:9931:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c110cb400/0x59bc99bb61948162 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x1daf3002fe9e0a3c expref: -99 pid: 9931 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1789760806.670987:0:9931:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c110cb400/0x59bc99bb61948162 lrc: 1/0,0 mode: --/PR res: [0x200000401:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x1daf3002fe9e0a3c expref: -99 pid: 9931 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1789760806.670991:0:9931:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1789760806.670993:0:9931:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c110cb400/0x59bc99bb61948162 lrc: 0/0,0 mode: --/PR res: [0x200000401:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x1daf3002fe9e0a3c expref: -99 pid: 9931 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000020:00100000:0.0:1789760806.671011:0:9931:0:(genops.c:2428:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffffa06c18a1e000: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:0.0:1789760806.671018:0:9931:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffffa06c091d2d80 x1876700024553088/t0(0) o36->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 488/512 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789760806.671031:0:9931:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d2d80 pname:cluuid:pid:xid:nid:opc:job chown:734aaee5-e21a-40f9-9d4e-90e3c76738d2:9931:1876700024553088:192.168.203.107@tcp:36:chown.0 00000100:00100000:0.0:1789760806.671061:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 11 seconds 00010000:00010000:1.0:1789760807.542985:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789760808.567014:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789760809.590935:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789760810.358999:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c09133000 lustre-OST0000_UUID: changing import state from FULL to CONNECTING 00000100:02000000:2.0:1789760810.359006:0:37:0:(import.c:2101:ptlrpc_disconnect_and_idle_import()) lustre-OST0000-osc-ffffa06c07e46000: disconnect after 21s idle 00000100:00100000:3.0:1789760810.359008:0:2343:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2343:1876700024553216:192.168.203.107@tcp:400:kworker.0 00000100:00100000:3.0:1789760810.359098:0:2343:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:b837c2fb-d1c1-4622-807e-a243f6fc2450:2343:1876700024553472:192.168.203.107@tcp:400:kworker.0 00000100:00100000:3.0:1789760810.359126:0:2343:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1789760810.359132:0:2343:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e53100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2343:1876700024553344:192.168.203.107@tcp:400:kworker.0 00000100:00080000:2.0:1789760810.361810:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c09134000 lustre-OST0001_UUID: changing import state from FULL to CONNECTING 00000100:02000000:2.0:1789760810.361813:0:37:0:(import.c:2101:ptlrpc_disconnect_and_idle_import()) lustre-OST0001-osc-ffffa06c07e46000: disconnect after 21s idle 00000100:00100000:3.0:1789760810.361873:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e52a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024553600:192.168.203.107@tcp:9:kworker.0 00000100:00100000:3.0:1789760810.361933:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e52d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024553728:192.168.203.107@tcp:9:kworker.0 00010000:00010000:1.0:1789760810.614975:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789760811.638975:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789760812.663003:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789760813.686949:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789760814.710999:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789760815.479055:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789760815.479060:0:8:0:(pinger.c:247:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1789760815.479063:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-OST0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789760815.479066:0:8:0:(pinger.c:247:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:1.0:1789760815.479085:0:2341:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2341:1876700024553856:192.168.203.107@tcp:400:kworker.0 00000100:00100000:0.0:1789760815.479088:0:2342:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2342:1876700024553984:192.168.203.107@tcp:400:kworker.0 00000100:00100000:1.0:1789760815.479186:0:2341:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:b837c2fb-d1c1-4622-807e-a243f6fc2450:2341:1876700024554112:192.168.203.107@tcp:400:kworker.0 00010000:00010000:1.0:1789760815.734964:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789760816.758970:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789760817.783018:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789760818.039061:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 1 seconds 00010000:00010000:1.0:1789760818.807116:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789760819.062954:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 1 seconds 00010000:00010000:0.0:1789760819.831055:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789760820.086961:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 1 seconds 00000100:00100000:3.0:1789760820.598939:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1789760810/real 1789760810] req@ffffa06c07e52a00 x1876700024553600/t0(0) o9->lustre-OST0000-osc-ffffa06c07e46000@192.168.203.107@tcp:28/4 lens 224/224 e 0 to 1 dl 1789760820 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789760820.599004:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e52a00 x1876700024553600/t0(0) o9->lustre-OST0000-osc-ffffa06c07e46000@192.168.203.107@tcp:28/4 lens 224/224 e 0 to 1 dl 1789760820 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789760820.599020:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1789760810/real 1789760810] req@ffffa06c07e52d80 x1876700024553728/t0(0) o9->lustre-OST0001-osc-ffffa06c07e46000@192.168.203.107@tcp:28/4 lens 224/224 e 0 to 1 dl 1789760820 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760820.599023:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789760820.599033:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1789760820.599040:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-OST0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789760820.599044:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:3.0:1789760820.599049:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e52d80 x1876700024553728/t0(0) o9->lustre-OST0001-osc-ffffa06c07e46000@192.168.203.107@tcp:28/4 lens 224/224 e 0 to 1 dl 1789760820 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789760820.599066:0:2342:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2342:1876700024554240:192.168.203.107@tcp:400:kworker.0 00000100:00100000:0.0:1789760820.599074:0:2341:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2341:1876700024554368:192.168.203.107@tcp:400:kworker.0 00000100:00080000:3.0:1789760820.599075:0:2340:0:(import.c:2011:ptlrpc_disconnect_idle_interpret()) @@@ inflight=1, refcount=3: rc = -110 req@ffffa06c07e52a00 x1876700024553600/t0(0) o9->lustre-OST0000-osc-ffffa06c07e46000@192.168.203.107@tcp:28/4 lens 224/224 e 0 to 1 dl 1789760820 ref 1 fl Interpret:EXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789760820.599089:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09133000 lustre-OST0000_UUID: changing import state from CONNECTING to IDLE 00000100:00100000:3.0:1789760820.599095:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e52a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024553600:192.168.203.107@tcp:9:kworker.0 00000100:00080000:3.0:1789760820.599104:0:2340:0:(import.c:2011:ptlrpc_disconnect_idle_interpret()) @@@ inflight=1, refcount=3: rc = -110 req@ffffa06c07e52d80 x1876700024553728/t0(0) o9->lustre-OST0001-osc-ffffa06c07e46000@192.168.203.107@tcp:28/4 lens 224/224 e 0 to 1 dl 1789760820 ref 1 fl Interpret:EXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789760820.599111:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09134000 lustre-OST0001_UUID: changing import state from CONNECTING to IDLE 00000100:00100000:3.0:1789760820.599115:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e52d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024553728:192.168.203.107@tcp:9:kworker.0 00000100:00100000:1.0:1789760820.599200:0:2342:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:b837c2fb-d1c1-4622-807e-a243f6fc2450:2342:1876700024554496:192.168.203.107@tcp:400:kworker.0 00010000:00010000:1.0:1789760820.855017:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789760821.110924:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 1 seconds 00010000:00010000:3.0:1789760821.879040:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000400:1.0:1789760822.135007:0:9931:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1789760806/real 1789760806] req@ffffa06c091d2d80 x1876700024553088/t0(0) o36->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 488/512 e 0 to 1 dl 1789760822 ref 2 fl Rpc:XQr/200/ffffffff rc 0/-1 job:'chown.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789760822.142600:0:9931:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from FULL to DISCONN 00000100:02000400:1.0:1789760822.142614:0:9931:0:(import.c:182:ptlrpc_set_import_discon()) lustre-MDT0000-mdc-ffffa06c18a1e000: Connection to lustre-MDT0000 (at 192.168.203.107@tcp) was lost; in progress operations using this service will wait for recovery to complete 00000100:00080000:1.0:1789760822.146365:0:9931:0:(import.c:418:ptlrpc_pinger_force()) lustre-MDT0000_UUID: waking up pinger s:DISCONN 00000100:00100000:1.0:1789760822.146383:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789760822.146505:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789760822.146510:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789760822.146514:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789760822.146539:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 184 00000100:00100000:1.0:1789760822.146540:0:2341:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2341:1876700024554624:192.168.203.107@tcp:400:kworker.0 00000100:00080000:0.0:1789760822.146564:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789760822.146580:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789760822.146601:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0380 x1876700024554752/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789760822.146668:0:2341:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1789760822.146681:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024554752:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789760822.146682:0:2341:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:b837c2fb-d1c1-4622-807e-a243f6fc2450:2341:1876700024554880:192.168.203.107@tcp:400:kworker.0 00010000:00010000:0.0:1789760822.902992:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789760823.158982:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789760823.927028:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789760824.182994:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789760824.950965:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789760825.206958:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789760825.975051:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789760826.231021:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00000400:3.0:1789760826.742935:0:2343:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1789760810/real 1789760810] req@ffffa06c07e53100 x1876700024553344/t0(0) o400->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 224/224 e 0 to 1 dl 1789760826 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789760826.752008:0:2343:0:(client.c:2547:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffffa06c07e53100 x1876700024553344/t0(0) o400->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 224/224 e 0 to 1 dl 1789760826 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789760826.752019:0:2343:0:(import.c:214:ptlrpc_set_import_discon()) mdc: import ffffa06c07e43000 already not connected (conn 2, was 1): CONNECTING 00000100:00000400:3.0:1789760826.752027:0:2343:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1789760810/real 1789760810] req@ffffa06c07e50700 x1876700024553216/t0(0) o400->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 224/224 e 0 to 1 dl 1789760826 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789760826.752038:0:2343:0:(client.c:2547:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffffa06c07e50700 x1876700024553216/t0(0) o400->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 224/224 e 0 to 1 dl 1789760826 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789760826.752045:0:2343:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from FULL to DISCONN 00000100:02020000:3.0:1789760826.752049:0:2343:0:(import.c:186:ptlrpc_set_import_discon()) MGC192.168.203.107@tcp: Connection to MGS (at 192.168.203.107@tcp) was lost; in progress operations using this service will fail 00000100:00080000:3.0:1789760826.755750:0:2343:0:(import.c:192:ptlrpc_set_import_discon()) import MGS@192.168.203.107@tcp for MGC192.168.203.107@tcp not replayable, auto-deactivating 00000100:00080000:3.0:1789760826.755753:0:2343:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00100000:3.0:1789760826.755760:0:2343:0:(client.c:3589:ptlrpc_abort_inflight()) @@@ inflight req@ffffa06c07e50700 x1876700024553216/t0(0) o400->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 224/224 e 0 to 1 dl 1789760826 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789760826.755773:0:2343:0:(client.c:3589:ptlrpc_abort_inflight()) @@@ inflight req@ffffa06c091d0a80 x1876700024553856/t0(0) o400->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 224/224 e 0 to 0 dl 1789760831 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789760826.755809:0:2343:0:(client.c:3589:ptlrpc_abort_inflight()) @@@ inflight req@ffffa06c091d2a00 x1876700024554240/t0(0) o400->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 224/224 e 0 to 0 dl 1789760836 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789760826.755823:0:2343:0:(client.c:3589:ptlrpc_abort_inflight()) @@@ inflight req@ffffa06c091d1f80 x1876700024554624/t0(0) o400->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 224/224 e 0 to 0 dl 1789760838 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 10000000:01000000:3.0:1789760826.755833:0:2343:0:(mgc_request.c:1168:mgc_import_event()) import event 0x808001 00000100:00080000:3.0:1789760826.755837:0:2343:0:(pinger.c:193:ptlrpc_pinger_ir_down()) IR down 10000000:01000000:3.0:1789760826.755838:0:2343:0:(mgc_request.c:1168:mgc_import_event()) import event 0x808002 00000100:00080000:3.0:1789760826.755840:0:2343:0:(import.c:418:ptlrpc_pinger_force()) MGS: waking up pinger s:DISCONN 00000100:00100000:0.0:1789760826.755910:0:2342:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2342:1876700024554240:192.168.203.107@tcp:400:kworker.0 00000100:00000400:3.0:1789760826.755923:0:2343:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1789760810/real 1789760810] req@ffffa06c07e50a80 x1876700024553472/t0(0) o400->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 224/224 e 0 to 1 dl 1789760826 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789760826.755940:0:2341:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2341:1876700024554624:192.168.203.107@tcp:400:kworker.0 00000100:00100000:1.0:1789760826.755977:0:2341:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2341:1876700024553856:192.168.203.107@tcp:400:kworker.0 00000100:00100000:3.0:1789760826.755977:0:2343:0:(client.c:2547:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffffa06c07e50a80 x1876700024553472/t0(0) o400->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 224/224 e 0 to 1 dl 1789760826 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760826.755986:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789760826.756003:0:2343:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from FULL to DISCONN 00000100:00080000:0.0:1789760826.756008:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:02000400:3.0:1789760826.756015:0:2343:0:(import.c:182:ptlrpc_set_import_discon()) lustre-MDT0000-mdc-ffffa06c07e46000: Connection to lustre-MDT0000 (at 192.168.203.107@tcp) was lost; in progress operations using this service will wait for recovery to complete 00000100:00080000:0.0:1789760826.756018:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789760826.756051:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 184 00000100:00080000:0.0:1789760826.756077:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789760826.756138:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d2a00 x1876700024555008/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789760826.756187:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789760826.756191:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:2.0:1789760826.756225:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024555008:192.168.203.107@tcp:250:kworker.0 00000100:00080000:3.0:1789760826.760581:0:2343:0:(import.c:418:ptlrpc_pinger_force()) lustre-MDT0000_UUID: waking up pinger s:DISCONN 00000100:00080000:0.0:1789760826.760591:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789760826.760597:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789760826.760600:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789760826.760608:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 228 00000100:00100000:3.0:1789760826.760608:0:2343:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e53100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2343:1876700024553344:192.168.203.107@tcp:400:kworker.0 00000100:00100000:3.0:1789760826.760621:0:2343:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2343:1876700024553216:192.168.203.107@tcp:400:kworker.0 00000100:00080000:0.0:1789760826.760622:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:3.0:1789760826.760637:0:2343:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:b837c2fb-d1c1-4622-807e-a243f6fc2450:2343:1876700024553472:192.168.203.107@tcp:400:kworker.0 00000008:00100000:0.0:1789760826.760641:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789760826.760669:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d1500 x1876700024555136/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789760826.760700:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789760826.760704:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:2.0:1789760826.760736:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024555136:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789760826.999091:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789760827.255917:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789760827.767057:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789760827.767072:0:8:0:(pinger.c:247:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1789760827.767078:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789760827.767083:0:8:0:(pinger.c:247:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1789760827.767090:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789760827.767095:0:8:0:(pinger.c:247:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00010000:00010000:2.0:1789760828.023025:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789760828.278996:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789760829.047022:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789760829.302931:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789760830.070990:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789760830.326981:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789760831.095108:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000400:1.0:1789760831.286951:0:2341:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1789760815/real 1789760815] req@ffffa06c091d3100 x1876700024554112/t0(0) o400->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 224/224 e 0 to 1 dl 1789760831 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789760831.303607:0:2341:0:(client.c:2547:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffffa06c091d3100 x1876700024554112/t0(0) o400->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 224/224 e 0 to 1 dl 1789760831 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789760831.303622:0:2341:0:(import.c:214:ptlrpc_set_import_discon()) mdc: import ffffa06c09137000 already not connected (conn 2, was 1): CONNECTING 00000100:00100000:1.0:1789760831.303659:0:2341:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:b837c2fb-d1c1-4622-807e-a243f6fc2450:2341:1876700024554112:192.168.203.107@tcp:400:kworker.0 00000100:00100000:1.0:1789760831.350924:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789760831.799084:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789760831.799097:0:8:0:(pinger.c:247:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1789760831.799103:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789760831.799108:0:8:0:(pinger.c:247:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1789760831.799115:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789760831.799120:0:8:0:(pinger.c:247:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000400:0.0:1789760831.863036:0:2342:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1789760815/real 1789760815] req@ffffa06c091d1180 x1876700024553984/t0(0) o400->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 224/224 e 0 to 1 dl 1789760831 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760831.863084:0:2342:0:(client.c:2547:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffffa06c091d1180 x1876700024553984/t0(0) o400->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 224/224 e 0 to 1 dl 1789760831 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760831.863101:0:2342:0:(import.c:214:ptlrpc_set_import_discon()) mdc: import ffffa06c07e43000 already not connected (conn 2, was 1): CONNECTING 00000100:00100000:0.0:1789760831.863120:0:2342:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2342:1876700024553984:192.168.203.107@tcp:400:kworker.0 00010000:00010000:2.0:1789760832.118973:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789760832.374951:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00100000:1.0:1789760832.887017:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1789760822/real 1789760822] req@ffffa06c091d0380 x1876700024554752/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760832 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789760832.887083:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0380 x1876700024554752/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760832 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789760832.887104:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789760832.887149:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789760832.887154:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024554752:192.168.203.107@tcp:38:kworker.0 00000100:00080000:0.0:1789760832.887181:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789760832.887200:0:8:0:(pinger.c:247:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1789760832.887205:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789760832.887209:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789760832.887213:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789760832.887221:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 261 00000100:00080000:0.0:1789760832.887233:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 5s 00000100:00080000:0.0:1789760832.887240:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789760832.887263:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789760832.887294:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d1180 x1876700024555264/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789760832.887323:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789760832.887327:0:8:0:(pinger.c:247:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:1.0:1789760832.887382:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024555264:192.168.203.107@tcp:38:kworker.0 00010000:00010000:0.0:1789760833.142982:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789760833.399100:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789760834.167048:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789760834.423010:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789760835.191092:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789760835.446958:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789760836.215055:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000400:1.0:1789760836.406992:0:2341:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1789760820/real 1789760820] req@ffffa06c091d2680 x1876700024554368/t0(0) o400->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 224/224 e 0 to 1 dl 1789760836 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789760836.429106:0:2341:0:(client.c:2547:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffffa06c091d2680 x1876700024554368/t0(0) o400->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 224/224 e 0 to 1 dl 1789760836 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789760836.429120:0:2341:0:(import.c:214:ptlrpc_set_import_discon()) mdc: import ffffa06c07e43000 already not connected (conn 3, was 1): CONNECTING 00000100:00100000:1.0:1789760836.429140:0:2341:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2341:1876700024554368:192.168.203.107@tcp:400:kworker.0 00000100:00100000:1.0:1789760836.470942:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00000400:0.0:1789760836.982970:0:2342:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1789760820/real 1789760820] req@ffffa06c091d3b80 x1876700024554496/t0(0) o400->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 224/224 e 0 to 1 dl 1789760836 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789760836.983006:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1789760826/real 1789760826] req@ffffa06c091d1500 x1876700024555136/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760836 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760836.983010:0:2342:0:(client.c:2547:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffffa06c091d3b80 x1876700024554496/t0(0) o400->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 224/224 e 0 to 1 dl 1789760836 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760836.983061:0:2342:0:(import.c:214:ptlrpc_set_import_discon()) mdc: import ffffa06c09137000 already not connected (conn 2, was 1): CONNECTING 00000100:00100000:1.0:1789760836.983072:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d1500 x1876700024555136/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760836 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760836.983088:0:2342:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:b837c2fb-d1c1-4622-807e-a243f6fc2450:2342:1876700024554496:192.168.203.107@tcp:400:kworker.0 00000100:00100000:1.0:1789760836.983090:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1789760826/real 1789760826] req@ffffa06c091d2a00 x1876700024555008/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789760836 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789760836.983115:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d2a00 x1876700024555008/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789760836 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789760836.983132:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789760836.983178:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789760836.983185:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024555136:192.168.203.107@tcp:38:kworker.0 00000100:00080000:1.0:1789760836.983194:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789760836.983202:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789760836.983206:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024555008:192.168.203.107@tcp:250:kworker.0 00000100:00080000:2.0:1789760836.983273:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789760836.983281:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789760836.983286:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789760836.983295:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 265 00000100:00080000:2.0:1789760836.983318:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 5s 00000100:00080000:2.0:1789760836.983327:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789760836.983359:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51c00 x1876700024555392/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789760836.983391:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789760836.983396:0:8:0:(pinger.c:247:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1789760836.983400:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789760836.983403:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789760836.983405:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789760836.983428:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 265 00000100:00080000:2.0:1789760836.983433:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 5s 00000100:00080000:2.0:1789760836.983456:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789760836.983467:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789760836.983482:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024555392:192.168.203.107@tcp:250:kworker.0 00000100:00100000:2.0:1789760836.983485:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e53480 x1876700024555520/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789760836.983608:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e53480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024555520:192.168.203.107@tcp:38:kworker.0 00010000:00010000:2.0:1789760837.239042:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789760837.494935:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789760838.006977:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789760838.006990:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1789760838.006995:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789760838.006999:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1789760838.007004:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789760838.007019:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00010000:00010000:2.0:1789760838.263092:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000400:1.0:1789760838.455021:0:2341:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1789760822/real 1789760822] req@ffffa06c091d0e00 x1876700024554880/t0(0) o400->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 224/224 e 0 to 1 dl 1789760838 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789760838.455065:0:2341:0:(client.c:2547:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffffa06c091d0e00 x1876700024554880/t0(0) o400->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 224/224 e 0 to 1 dl 1789760838 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789760838.455075:0:2341:0:(import.c:214:ptlrpc_set_import_discon()) mdc: import ffffa06c09137000 already not connected (conn 3, was 1): CONNECTING 00000100:00100000:1.0:1789760838.455095:0:2341:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:b837c2fb-d1c1-4622-807e-a243f6fc2450:2341:1876700024554880:192.168.203.107@tcp:400:kworker.0 00000100:00100000:1.0:1789760838.518981:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789760839.287070:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789760839.543021:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789760840.311127:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789760840.566927:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789760841.335083:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789760841.590951:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789760842.039185:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789760842.039201:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1789760842.039205:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789760842.039210:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1789760842.039215:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789760842.039219:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00010000:00010000:0.0:1789760842.359039:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789760842.614903:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789760843.383055:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789760843.638908:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789760844.406991:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789760844.662915:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789760845.431130:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789760845.686975:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789760846.455012:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789760846.710967:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00100000:1.0:1789760847.222927:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1789760832/real 1789760832] req@ffffa06c091d1180 x1876700024555264/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760847 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789760847.222953:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d1180 x1876700024555264/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760847 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760847.222977:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789760847.222978:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760847.222985:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1789760847.222988:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789760847.222991:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789760847.222993:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789760847.222997:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00080000:0.0:1789760847.223002:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 271 00000100:00100000:1.0:1789760847.223004:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024555264:192.168.203.107@tcp:38:kworker.0 00000100:00080000:0.0:1789760847.223029:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 10s 00000100:00080000:0.0:1789760847.223037:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789760847.223046:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789760847.223071:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d3b80 x1876700024555648/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789760847.223093:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789760847.223096:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1789760847.223104:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789760847.223107:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1789760847.223111:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789760847.223113:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:1.0:1789760847.223121:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024555648:192.168.203.107@tcp:38:kworker.0 00010000:00010000:0.0:1789760847.478974:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789760847.734911:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789760848.503094:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789760848.759917:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789760849.527021:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789760849.782940:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789760850.551066:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789760850.806942:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00100000:1.0:1789760851.319013:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1789760836/real 1789760836] req@ffffa06c07e53480 x1876700024555520/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760851 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789760851.319073:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e53480 x1876700024555520/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760851 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789760851.319087:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1789760836/real 1789760836] req@ffffa06c07e51c00 x1876700024555392/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789760851 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789760851.319103:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51c00 x1876700024555392/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789760851 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789760851.319125:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789760851.319165:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789760851.319171:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e53480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024555520:192.168.203.107@tcp:38:kworker.0 00000100:00080000:1.0:1789760851.319204:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789760851.319213:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789760851.319218:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024555392:192.168.203.107@tcp:250:kworker.0 00000100:00080000:0.0:1789760851.319270:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789760851.319283:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789760851.319290:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789760851.319330:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 275 00000100:00080000:0.0:1789760851.319364:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 10s 00000100:00080000:0.0:1789760851.319376:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789760851.319425:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0000 x1876700024555776/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789760851.319481:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789760851.319487:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1789760851.319504:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789760851.319509:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789760851.319513:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789760851.319520:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 275 00000100:00080000:0.0:1789760851.319528:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 10s 00000100:00100000:1.0:1789760851.319539:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024555776:192.168.203.107@tcp:250:kworker.0 00000100:00080000:0.0:1789760851.319541:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789760851.319562:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789760851.319589:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d2300 x1876700024555904/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789760851.319674:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024555904:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789760851.575032:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789760851.830955:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789760852.342919:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789760852.342930:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1789760852.342934:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789760852.342939:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1789760852.342946:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789760852.342951:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00010000:00010000:3.0:1789760852.599076:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789760852.855044:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789760853.623125:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789760853.878964:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789760854.647132:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789760854.903165:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789760855.670995:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789760855.926971:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:1.0:1789760856.375108:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789760856.375121:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1789760856.375126:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789760856.375131:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1789760856.375137:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789760856.375731:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00010000:00010000:2.0:1789760856.695035:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789760856.950906:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789760857.719025:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789760857.974908:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789760858.743011:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789760858.998959:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789760859.766989:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789760860.022926:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789760860.791071:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789760861.046926:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:1.0:1789760861.559004:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789760861.559022:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1789760861.559030:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789760861.559038:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1789760861.559046:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789760861.559053:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00010000:00010000:3.0:1789760861.814973:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789760862.070955:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789760862.839041:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789760863.094981:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789760863.863001:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789760864.118948:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789760864.886948:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789760865.142908:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789760865.911091:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789760866.167924:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:1.0:1789760866.678969:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789760866.678986:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1789760866.678992:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789760866.678999:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1789760866.679006:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789760866.679012:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000800:00000100:2.0:1789760866.935007:0:2332:0:(socklnd_cb.c:2254:ksocknal_find_timed_out_conn()) A connection with 12345-192.168.203.107@tcp (192.168.203.107:988) timed out; the network or node may be down. 00010000:00010000:0.0:1789760866.935055:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789760867.190957:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00100000:2.0:1789760867.702968:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1789760847/real 1789760847] req@ffffa06c091d3b80 x1876700024555648/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760867 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789760867.703025:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d3b80 x1876700024555648/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760867 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789760867.703053:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789760867.703083:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789760867.703089:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024555648:192.168.203.107@tcp:38:kworker.0 00000100:00080000:1.0:1789760867.703144:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789760867.703153:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1789760867.703158:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789760867.703194:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789760867.703199:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789760867.703208:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 286 00000100:00080000:1.0:1789760867.703220:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 15s 00000100:00080000:1.0:1789760867.703228:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789760867.703241:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789760867.703303:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2e300 x1876700024556032/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789760867.703348:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789760867.703356:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:2.0:1789760867.703415:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024556032:192.168.203.107@tcp:38:kworker.0 00010000:00010000:1.0:1789760867.959052:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789760868.214913:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789760868.983071:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789760869.238947:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789760870.007144:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1789760870.007947:0:2330:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1021 -> 192.168.203.107:988 00000400:00000100:1.0:1789760870.008055:0:2330:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.107@tcp at host 192.168.203.107 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1789760870.008083:0:2330:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.107@tcp 00000100:00100000:2.0:1789760870.008332:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760867/real 1789760870] req@ffffa06c08e2e300 x1876700024556032/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760892 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789760870.008393:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2e300 x1876700024556032/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760892 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789760870.008416:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789760870.008426:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789760870.008434:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024556032:192.168.203.107@tcp:38:kworker.0 00000100:00100000:2.0:1789760870.262919:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789760871.031041:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789760871.286944:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00100000:2.0:1789760871.798929:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1789760851/real 1789760851] req@ffffa06c091d2300 x1876700024555904/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760871 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789760871.798959:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d2300 x1876700024555904/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760871 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789760871.798991:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1789760851/real 1789760851] req@ffffa06c091d0000 x1876700024555776/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789760871 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789760871.799001:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0000 x1876700024555776/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789760871 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789760871.799016:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789760871.799044:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789760871.799047:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024555904:192.168.203.107@tcp:38:kworker.0 00000100:00080000:2.0:1789760871.799052:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789760871.799055:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789760871.799057:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024555776:192.168.203.107@tcp:250:kworker.0 00000100:00080000:1.0:1789760871.799076:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789760871.799081:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789760871.799084:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789760871.799090:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 290 00000100:00080000:1.0:1789760871.799110:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 15s 00000100:00080000:1.0:1789760871.799116:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789760871.799142:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2c000 x1876700024556160/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789760871.799175:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789760871.799179:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789760871.799181:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789760871.799186:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 306 00000100:00080000:1.0:1789760871.799190:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789760871.799206:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789760871.799216:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789760871.799232:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2d500 x1876700024556288/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789760871.799235:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024556160:192.168.203.107@tcp:250:kworker.0 00000100:00080000:1.0:1789760871.799251:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789760871.799255:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789760871.799257:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789760871.799268:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 290 00000100:00080000:1.0:1789760871.799276:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 15s 00000100:00080000:1.0:1789760871.799289:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789760871.799294:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789760871.799305:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2e300 x1876700024556416/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789760871.799348:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024556288:192.168.203.107@tcp:38:kworker.0 00000100:00100000:2.0:1789760871.799377:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024556416:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789760872.054978:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789760872.310918:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789760873.078931:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1789760873.079103:0:2331:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1021 -> 192.168.203.107:988 00000400:00000100:3.0:1789760873.079260:0:2331:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.107@tcp at host 192.168.203.107 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1789760873.079276:0:2331:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.107@tcp 00000800:00000100:3.0:1789760873.079320:0:2331:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.107@tcp 00000800:00000100:3.0:1789760873.079332:0:2331:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.107@tcp 00000100:00100000:2.0:1789760873.079415:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760871/real 1789760873] req@ffffa06c08e2d500 x1876700024556288/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760896 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789760873.079444:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2d500 x1876700024556288/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760896 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789760873.079458:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789760873.079461:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789760873.079472:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024556288:192.168.203.107@tcp:38:kworker.0 00000100:00100000:2.0:1789760873.079478:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760871/real 1789760873] req@ffffa06c08e2e300 x1876700024556416/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760896 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789760873.079488:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2e300 x1876700024556416/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760896 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789760873.079493:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789760873.079495:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789760873.079498:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024556416:192.168.203.107@tcp:38:kworker.0 00000100:00100000:2.0:1789760873.079505:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760871/real 1789760873] req@ffffa06c08e2c000 x1876700024556160/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789760896 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789760873.079517:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2c000 x1876700024556160/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789760896 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789760873.079526:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789760873.079529:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789760873.079532:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024556160:192.168.203.107@tcp:250:kworker.0 00000100:00100000:2.0:1789760873.334909:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789760874.103058:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789760874.358930:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789760875.126963:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789760875.382910:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789760876.151009:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789760876.406928:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:1.0:1789760876.918958:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789760876.918988:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789760876.918994:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789760876.919008:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 310 00000100:00080000:1.0:1789760876.919022:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789760876.919030:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789760876.919076:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2ca80 x1876700024556544/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789760876.919130:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789760876.919136:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789760876.919138:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789760876.919145:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 310 00000100:00080000:1.0:1789760876.919151:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789760876.919158:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789760876.919172:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789760876.919196:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024556544:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789760876.919210:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2d180 x1876700024556672/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789760876.919239:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789760876.919244:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789760876.919261:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789760876.919268:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 310 00000100:00080000:1.0:1789760876.919297:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789760876.919303:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789760876.919312:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789760876.919336:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2c700 x1876700024556800/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789760876.919368:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024556672:192.168.203.107@tcp:38:kworker.0 00000100:00100000:2.0:1789760876.919431:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024556800:192.168.203.107@tcp:38:kworker.0 00010000:00010000:0.0:1789760877.175144:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789760877.430960:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789760878.198942:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789760878.454947:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789760879.222944:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789760879.478912:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000400:00000100:3.0:1789760879.991008:0:2328:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1021 -> 192.168.203.107:988 00000400:00000100:3.0:1789760879.991043:0:2328:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.107@tcp at host 192.168.203.107 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1789760879.991057:0:2328:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.107@tcp 00000800:00000100:3.0:1789760879.991084:0:2328:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.107@tcp 00000800:00000100:3.0:1789760879.991105:0:2328:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.107@tcp 00000100:00100000:2.0:1789760879.991154:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760876/real 1789760879] req@ffffa06c08e2c700 x1876700024556800/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760901 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789760879.991181:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2c700 x1876700024556800/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760901 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789760879.991193:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789760879.991198:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789760879.991203:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024556800:192.168.203.107@tcp:38:kworker.0 00000100:00100000:2.0:1789760879.991211:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760876/real 1789760879] req@ffffa06c08e2d180 x1876700024556672/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760901 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789760879.991236:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2d180 x1876700024556672/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760901 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789760879.991245:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789760879.991249:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789760879.991252:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024556672:192.168.203.107@tcp:38:kworker.0 00000100:00100000:2.0:1789760879.991259:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760876/real 1789760879] req@ffffa06c08e2ca80 x1876700024556544/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789760901 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789760879.991272:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2ca80 x1876700024556544/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789760901 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789760879.991280:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789760879.991292:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789760879.991296:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024556544:192.168.203.107@tcp:250:kworker.0 00010000:00010000:3.0:1789760880.246989:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789760880.502965:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789760881.270965:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789760881.526957:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789760882.038937:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789760882.038946:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789760882.038960:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789760882.038969:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 315 00000100:00080000:0.0:1789760882.038977:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789760882.038983:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789760882.039013:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0000 x1876700024556928/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789760882.039035:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789760882.039037:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789760882.039039:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789760882.039042:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 315 00000100:00080000:0.0:1789760882.039044:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789760882.039046:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789760882.039054:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789760882.039063:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d2300 x1876700024557056/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789760882.039084:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789760882.039088:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789760882.039090:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789760882.039105:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 315 00000100:00080000:0.0:1789760882.039111:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1789760882.039111:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024556928:192.168.203.107@tcp:250:kworker.0 00000100:00080000:0.0:1789760882.039120:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789760882.039127:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789760882.039142:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d3b80 x1876700024557184/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789760882.039193:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789760882.039234:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760882/real 1789760882] req@ffffa06c091d0000 x1876700024556928/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789760907 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789760882.039264:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0000 x1876700024556928/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789760907 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789760882.039272:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789760882.039275:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789760882.039278:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024556928:192.168.203.107@tcp:250:kworker.0 00000100:00100000:2.0:1789760882.039285:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024557056:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789760882.039318:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789760882.039332:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760882/real 1789760882] req@ffffa06c091d2300 x1876700024557056/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760907 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789760882.039346:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d2300 x1876700024557056/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760907 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789760882.039355:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789760882.039358:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789760882.039362:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024557056:192.168.203.107@tcp:38:kworker.0 00000100:00100000:2.0:1789760882.039398:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024557184:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789760882.039429:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789760882.039442:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760882/real 1789760882] req@ffffa06c091d3b80 x1876700024557184/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760907 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789760882.039454:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d3b80 x1876700024557184/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760907 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789760882.039469:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789760882.039473:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789760882.039477:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024557184:192.168.203.107@tcp:38:kworker.0 00010000:00010000:1.0:1789760882.294952:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789760882.550913:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789760883.318989:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789760883.574925:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789760884.342983:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789760884.598907:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789760885.366963:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789760885.622901:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789760886.390958:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789760886.646918:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:1.0:1789760887.158972:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789760887.158980:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789760887.158983:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789760887.158992:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 320 00000100:00080000:1.0:1789760887.158999:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789760887.159005:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789760887.159048:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2ed80 x1876700024557312/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789760887.159070:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789760887.159073:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789760887.159075:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789760887.159078:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 320 00000100:00080000:1.0:1789760887.159080:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789760887.159083:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789760887.159090:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789760887.159111:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2ca80 x1876700024557440/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789760887.159120:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789760887.159123:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789760887.159124:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789760887.159126:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 320 00000100:00080000:1.0:1789760887.159130:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1789760887.159132:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024557312:192.168.203.107@tcp:250:kworker.0 00000100:00080000:1.0:1789760887.159135:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789760887.159142:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789760887.159152:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2d180 x1876700024557568/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789760887.159222:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024557440:192.168.203.107@tcp:38:kworker.0 00000100:00100000:2.0:1789760887.159260:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024557568:192.168.203.107@tcp:38:kworker.0 00010000:00010000:1.0:1789760887.414957:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789760887.670905:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789760888.438948:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789760888.694916:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000400:00000100:3.0:1789760889.207153:0:2329:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1021 -> 192.168.203.107:988 00000400:00000100:3.0:1789760889.207192:0:2329:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.107@tcp at host 192.168.203.107 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1789760889.207205:0:2329:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.107@tcp 00000800:00000100:3.0:1789760889.207236:0:2329:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.107@tcp 00000800:00000100:3.0:1789760889.207242:0:2329:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.107@tcp 00000100:00100000:2.0:1789760889.207286:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760887/real 1789760889] req@ffffa06c08e2d180 x1876700024557568/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760912 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789760889.207312:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2d180 x1876700024557568/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760912 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789760889.207344:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789760889.207350:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789760889.207354:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024557568:192.168.203.107@tcp:38:kworker.0 00000100:00100000:2.0:1789760889.207360:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760887/real 1789760889] req@ffffa06c08e2ed80 x1876700024557312/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789760912 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789760889.207371:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2ed80 x1876700024557312/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789760912 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789760889.207379:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789760889.207388:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789760889.207392:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024557312:192.168.203.107@tcp:250:kworker.0 00000100:00100000:2.0:1789760889.207400:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760887/real 1789760889] req@ffffa06c08e2ca80 x1876700024557440/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760912 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789760889.207412:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2ca80 x1876700024557440/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760912 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789760889.207421:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789760889.207424:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789760889.207427:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024557440:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789760889.463014:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789760889.718906:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789760890.486952:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789760890.742925:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789760891.510973:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789760891.766917:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:3.0:1789760892.278919:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789760892.278928:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789760892.278931:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789760892.278949:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 326 00000100:00080000:3.0:1789760892.278958:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789760892.278964:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:3.0:1789760892.278991:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872ce00 x1876700024557696/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789760892.279023:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789760892.279026:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789760892.279027:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789760892.279030:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 326 00000100:00080000:3.0:1789760892.279033:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789760892.279037:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789760892.279056:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789760892.279065:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024557696:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789760892.279074:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872d880 x1876700024557824/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789760892.279089:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789760892.279094:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789760892.279095:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789760892.279098:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 326 00000100:00080000:3.0:1789760892.279102:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789760892.279105:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789760892.279110:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789760892.279119:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872f800 x1876700024557952/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789760892.279135:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789760892.279183:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760892/real 1789760892] req@ffffa06c1872ce00 x1876700024557696/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789760917 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789760892.279203:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872ce00 x1876700024557696/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789760917 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789760892.279215:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789760892.279219:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789760892.279222:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024557696:192.168.203.107@tcp:250:kworker.0 00000100:00100000:2.0:1789760892.279241:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024557824:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789760892.279269:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789760892.279288:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760892/real 1789760892] req@ffffa06c1872d880 x1876700024557824/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760917 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789760892.279302:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872d880 x1876700024557824/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760917 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789760892.279312:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789760892.279315:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789760892.279319:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024557824:192.168.203.107@tcp:38:kworker.0 00000100:00100000:2.0:1789760892.279326:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024557952:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789760892.279344:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789760892.279355:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760892/real 1789760892] req@ffffa06c1872f800 x1876700024557952/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760917 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789760892.279368:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872f800 x1876700024557952/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760917 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789760892.279401:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789760892.279404:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789760892.279407:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024557952:192.168.203.107@tcp:38:kworker.0 00010000:00010000:2.0:1789760892.535031:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789760892.790918:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789760893.559000:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789760893.815030:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789760894.582963:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789760894.838906:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789760895.606997:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789760895.862988:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789760896.630936:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789760896.886982:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:1.0:1789760897.398913:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789760897.398921:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789760897.398924:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789760897.398932:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 331 00000100:00080000:1.0:1789760897.398940:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789760897.398945:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789760897.398989:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2c700 x1876700024558080/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789760897.399027:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789760897.399031:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789760897.399033:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789760897.399038:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 331 00000100:00080000:1.0:1789760897.399043:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789760897.399049:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789760897.399057:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789760897.399072:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024558080:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789760897.399073:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2c000 x1876700024558208/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789760897.399089:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789760897.399091:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789760897.399093:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789760897.399096:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 331 00000100:00080000:1.0:1789760897.399099:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789760897.399101:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789760897.399105:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789760897.399114:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2e300 x1876700024558336/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789760897.399165:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024558208:192.168.203.107@tcp:38:kworker.0 00000100:00100000:2.0:1789760897.399188:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024558336:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789760897.654934:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789760897.911043:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000400:00000100:2.0:1789760897.911145:0:2330:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1021 -> 192.168.203.107:988 00000400:00000100:2.0:1789760897.911191:0:2330:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.107@tcp at host 192.168.203.107 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1789760897.911213:0:2330:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.107@tcp 00000800:00000100:2.0:1789760897.911244:0:2330:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.107@tcp 00000800:00000100:2.0:1789760897.911252:0:2330:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.107@tcp 00000100:00100000:1.0:1789760897.911273:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760897/real 1789760897] req@ffffa06c08e2c000 x1876700024558208/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760922 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789760897.911300:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2c000 x1876700024558208/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760922 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789760897.911319:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789760897.911322:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789760897.911326:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024558208:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789760897.911332:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760897/real 1789760897] req@ffffa06c08e2e300 x1876700024558336/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760922 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789760897.911341:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2e300 x1876700024558336/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760922 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789760897.911347:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789760897.911350:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789760897.911352:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024558336:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789760897.911356:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760897/real 1789760897] req@ffffa06c08e2c700 x1876700024558080/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789760922 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789760897.911364:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2c700 x1876700024558080/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789760922 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789760897.911368:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789760897.911370:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789760897.911372:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024558080:192.168.203.107@tcp:250:kworker.0 00010000:00010000:3.0:1789760898.679203:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789760898.934923:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789760899.703149:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789760899.958988:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789760900.727006:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789760900.982906:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789760901.751103:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789760902.007048:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789760902.518957:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789760902.518972:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789760902.518978:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789760902.518986:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 336 00000100:00080000:2.0:1789760902.518995:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789760902.519000:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789760902.519031:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e53800 x1876700024558464/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789760902.519050:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789760902.519069:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789760902.519071:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789760902.519076:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 336 00000100:00080000:2.0:1789760902.519081:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789760902.519086:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789760902.519094:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789760902.519096:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e53800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024558464:192.168.203.107@tcp:250:kworker.0 00000100:00100000:2.0:1789760902.519107:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e52680 x1876700024558592/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789760902.519120:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789760902.519123:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789760902.519124:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789760902.519128:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 336 00000100:00080000:2.0:1789760902.519131:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789760902.519134:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789760902.519138:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789760902.519148:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51f80 x1876700024558720/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789760902.519162:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789760902.519199:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760902/real 1789760902] req@ffffa06c07e53800 x1876700024558464/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789760927 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789760902.519217:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e53800 x1876700024558464/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789760927 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789760902.519228:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789760902.519232:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789760902.519237:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e53800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024558464:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789760902.519260:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e52680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024558592:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789760902.519285:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789760902.519298:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760902/real 1789760902] req@ffffa06c07e52680 x1876700024558592/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760927 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789760902.519309:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e52680 x1876700024558592/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760927 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789760902.519315:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789760902.519317:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789760902.519319:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e52680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024558592:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789760902.519324:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024558720:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789760902.519338:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789760902.519345:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760902/real 1789760902] req@ffffa06c07e51f80 x1876700024558720/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760927 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789760902.519363:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51f80 x1876700024558720/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760927 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789760902.519368:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789760902.519370:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789760902.519372:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024558720:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789760902.775096:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789760903.030951:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789760903.799026:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789760904.055029:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789760904.822984:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789760905.078920:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789760905.847036:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789760906.103024:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789760906.871096:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789760907.126973:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:1.0:1789760907.638980:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789760907.638991:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789760907.638995:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789760907.639005:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 341 00000100:00080000:1.0:1789760907.639016:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789760907.639022:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789760907.639059:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2c700 x1876700024558848/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789760907.639088:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789760907.639092:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789760907.639095:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789760907.639099:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 341 00000100:00080000:1.0:1789760907.639101:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789760907.639122:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789760907.639131:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789760907.639146:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2e300 x1876700024558976/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789760907.639162:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789760907.639165:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789760907.639168:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1789760907.639177:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024558848:192.168.203.107@tcp:250:kworker.0 00000100:00080000:1.0:1789760907.639177:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 341 00000100:00080000:1.0:1789760907.639188:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789760907.639197:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789760907.639202:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789760907.639216:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2c000 x1876700024559104/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789760907.639289:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789760907.639355:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760907/real 1789760907] req@ffffa06c08e2c700 x1876700024558848/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789760932 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760907.639382:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2c700 x1876700024558848/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789760932 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760907.639405:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760907.639410:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789760907.639414:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024558848:192.168.203.107@tcp:250:kworker.0 00000100:00100000:0.0:1789760907.639442:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024558976:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789760907.639466:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789760907.639482:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760907/real 1789760907] req@ffffa06c08e2e300 x1876700024558976/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760932 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760907.639492:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2e300 x1876700024558976/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760932 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760907.639498:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760907.639501:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789760907.639503:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024558976:192.168.203.107@tcp:38:kworker.0 00000100:00100000:0.0:1789760907.639507:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024559104:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789760907.639536:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789760907.639543:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760907/real 1789760907] req@ffffa06c08e2c000 x1876700024559104/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760932 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760907.639551:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2c000 x1876700024559104/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760932 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760907.639557:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760907.639559:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789760907.639561:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024559104:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789760907.895057:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789760908.151094:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789760908.919042:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789760909.174957:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789760909.943020:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789760910.198982:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789760910.967024:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789760911.223045:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789760911.990999:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789760912.246978:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789760912.758960:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789760912.758968:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789760912.758971:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789760912.758979:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 346 00000100:00080000:2.0:1789760912.758987:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789760912.758993:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789760912.759023:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50380 x1876700024559232/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789760912.759056:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789760912.759059:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789760912.759060:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789760912.759064:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 346 00000100:00080000:2.0:1789760912.759066:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789760912.759069:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789760912.759076:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789760912.759087:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e53b80 x1876700024559360/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789760912.759097:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1789760912.759098:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024559232:192.168.203.107@tcp:250:kworker.0 00000100:00080000:2.0:1789760912.759101:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789760912.759105:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789760912.759109:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 346 00000100:00080000:2.0:1789760912.759112:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789760912.759114:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789760912.759117:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789760912.759123:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50e00 x1876700024559488/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789760912.759162:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789760912.759193:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760912/real 1789760912] req@ffffa06c07e50380 x1876700024559232/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789760937 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760912.759206:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50380 x1876700024559232/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789760937 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760912.759214:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760912.759217:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789760912.759220:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024559232:192.168.203.107@tcp:250:kworker.0 00000100:00100000:0.0:1789760912.759244:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e53b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024559360:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789760912.759262:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789760912.759274:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760912/real 1789760912] req@ffffa06c07e53b80 x1876700024559360/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760937 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760912.759303:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e53b80 x1876700024559360/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760937 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760912.759313:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760912.759316:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789760912.759318:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e53b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024559360:192.168.203.107@tcp:38:kworker.0 00000100:00100000:0.0:1789760912.759323:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024559488:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789760912.759340:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789760912.759348:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760912/real 1789760912] req@ffffa06c07e50e00 x1876700024559488/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760937 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760912.759355:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50e00 x1876700024559488/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760937 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760912.759361:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760912.759363:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789760912.759365:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024559488:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789760913.015012:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789760913.270913:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789760914.038967:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789760914.294928:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789760915.063011:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789760915.318966:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789760916.086971:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789760916.342952:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789760917.111108:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789760917.366956:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789760917.878972:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789760917.878982:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789760917.878987:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789760917.878998:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 351 00000100:00080000:2.0:1789760917.879027:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789760917.879037:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789760917.879086:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51880 x1876700024559616/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789760917.879114:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789760917.879119:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789760917.879121:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789760917.879126:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 351 00000100:00080000:2.0:1789760917.879130:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789760917.879136:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789760917.879146:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789760917.879162:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51500 x1876700024559744/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789760917.879178:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024559616:192.168.203.107@tcp:250:kworker.0 00000100:00080000:2.0:1789760917.879182:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789760917.879192:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789760917.879195:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789760917.879202:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 351 00000100:00080000:2.0:1789760917.879206:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789760917.879213:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789760917.879220:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789760917.879241:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50000 x1876700024559872/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789760917.879327:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024559744:192.168.203.107@tcp:38:kworker.0 00000100:00100000:0.0:1789760917.879358:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024559872:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789760918.134960:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789760918.390966:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789760919.158943:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789760919.414981:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789760920.183003:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789760920.438945:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000400:00000100:3.0:1789760920.951062:0:2331:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.107:988 00000400:00000100:3.0:1789760920.951107:0:2331:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.107@tcp at host 192.168.203.107 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1789760920.951126:0:2331:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.107@tcp 00000800:00000100:3.0:1789760920.951170:0:2331:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.107@tcp 00000800:00000100:3.0:1789760920.951181:0:2331:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.107@tcp 00000100:00100000:0.0:1789760920.951211:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760917/real 1789760920] req@ffffa06c07e51500 x1876700024559744/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760942 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760920.951278:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51500 x1876700024559744/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760942 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760920.951292:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760920.951298:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789760920.951304:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024559744:192.168.203.107@tcp:38:kworker.0 00000100:00100000:0.0:1789760920.951314:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760917/real 1789760920] req@ffffa06c07e50000 x1876700024559872/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760942 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760920.951329:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50000 x1876700024559872/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760942 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760920.951337:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760920.951341:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789760920.951344:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024559872:192.168.203.107@tcp:38:kworker.0 00000100:00100000:0.0:1789760920.951352:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760917/real 1789760920] req@ffffa06c07e51880 x1876700024559616/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789760942 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760920.951364:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51880 x1876700024559616/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789760942 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760920.951373:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760920.951377:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789760920.951381:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024559616:192.168.203.107@tcp:250:kworker.0 00010000:00010000:3.0:1789760921.206988:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789760921.462913:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789760922.230972:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789760922.486888:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789760922.999095:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789760922.999106:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789760922.999109:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789760922.999136:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 356 00000100:00080000:0.0:1789760922.999159:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789760922.999168:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789760922.999241:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d1180 x1876700024560000/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789760922.999270:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789760922.999275:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789760922.999277:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789760922.999282:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 356 00000100:00080000:0.0:1789760922.999286:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789760922.999292:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789760922.999302:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789760922.999310:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024560000:192.168.203.107@tcp:250:kworker.0 00000100:00100000:0.0:1789760922.999319:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0e00 x1876700024560128/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789760922.999334:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789760922.999355:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789760922.999368:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789760922.999374:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 356 00000100:00080000:0.0:1789760922.999380:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789760922.999386:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789760922.999394:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1789760922.999429:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789760922.999439:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d2a00 x1876700024560256/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789760922.999472:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760922/real 1789760922] req@ffffa06c091d1180 x1876700024560000/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789760947 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789760922.999522:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d1180 x1876700024560000/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789760947 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789760922.999537:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789760922.999557:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789760922.999561:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024560000:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789760922.999586:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024560128:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789760922.999611:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789760922.999624:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760922/real 1789760922] req@ffffa06c091d0e00 x1876700024560128/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760947 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789760922.999648:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0e00 x1876700024560128/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760947 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789760922.999657:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789760922.999661:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789760922.999665:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024560128:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789760922.999672:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024560256:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789760922.999693:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789760922.999706:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760922/real 1789760922] req@ffffa06c091d2a00 x1876700024560256/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760947 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789760922.999717:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d2a00 x1876700024560256/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760947 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789760922.999738:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789760922.999742:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789760922.999745:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024560256:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789760923.255006:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789760923.510901:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789760924.279057:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789760924.534926:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789760925.302963:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789760925.558941:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789760926.326978:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789760926.583893:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789760927.351026:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789760927.606940:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789760928.118970:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789760928.118980:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789760928.118984:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789760928.118995:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 361 00000100:00080000:2.0:1789760928.119007:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789760928.119015:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789760928.119055:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e52300 x1876700024560384/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789760928.119083:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789760928.119086:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789760928.119088:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789760928.119091:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 361 00000100:00080000:2.0:1789760928.119094:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789760928.119097:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789760928.119106:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789760928.119116:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51180 x1876700024560512/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789760928.119127:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789760928.119136:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:1.0:1789760928.119138:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e52300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024560384:192.168.203.107@tcp:250:kworker.0 00000100:00080000:2.0:1789760928.119138:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789760928.119156:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 361 00000100:00080000:2.0:1789760928.119159:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789760928.119163:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789760928.119181:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789760928.119197:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51880 x1876700024560640/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789760928.119233:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789760928.119274:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760928/real 1789760928] req@ffffa06c07e52300 x1876700024560384/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789760953 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789760928.119295:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e52300 x1876700024560384/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789760953 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789760928.119308:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789760928.119313:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789760928.119318:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e52300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024560384:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789760928.119347:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024560512:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789760928.119367:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789760928.119381:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760928/real 1789760928] req@ffffa06c07e51180 x1876700024560512/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760953 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789760928.119394:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51180 x1876700024560512/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760953 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789760928.119420:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789760928.119423:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789760928.119426:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024560512:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789760928.119431:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024560640:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789760928.119449:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789760928.119457:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760928/real 1789760928] req@ffffa06c07e51880 x1876700024560640/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760953 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789760928.119465:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51880 x1876700024560640/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760953 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789760928.119471:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789760928.119473:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789760928.119475:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024560640:192.168.203.107@tcp:38:kworker.0 00010000:00010000:2.0:1789760928.375004:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789760928.630915:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789760929.398996:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789760929.654927:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789760930.422977:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789760930.678910:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789760931.446969:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789760931.702898:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789760932.471022:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789760932.726910:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789760933.238975:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789760933.238985:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789760933.238989:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789760933.238999:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 367 00000100:00080000:2.0:1789760933.239011:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789760933.239018:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789760933.239054:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50000 x1876700024560768/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789760933.239081:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789760933.239085:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789760933.239087:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789760933.239091:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 367 00000100:00080000:2.0:1789760933.239095:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789760933.239100:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789760933.239109:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789760933.239121:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51500 x1876700024560896/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789760933.239135:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789760933.239152:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789760933.239154:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789760933.239159:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 367 00000100:00080000:2.0:1789760933.239164:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789760933.239171:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789760933.239178:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024560768:192.168.203.107@tcp:250:kworker.0 00000008:00100000:2.0:1789760933.239182:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789760933.239203:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50e00 x1876700024561024/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789760933.239269:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789760933.239314:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760933/real 1789760933] req@ffffa06c07e50000 x1876700024560768/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789760958 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789760933.239329:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50000 x1876700024560768/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789760958 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789760933.239351:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789760933.239355:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789760933.239359:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024560768:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789760933.239365:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024560896:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789760933.239383:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789760933.239392:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760933/real 1789760933] req@ffffa06c07e51500 x1876700024560896/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760958 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789760933.239400:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51500 x1876700024560896/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760958 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789760933.239405:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789760933.239407:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789760933.239409:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024560896:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789760933.239433:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024561024:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789760933.239445:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789760933.239451:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760933/real 1789760933] req@ffffa06c07e50e00 x1876700024561024/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760958 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789760933.239458:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50e00 x1876700024561024/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760958 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789760933.239464:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789760933.239465:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789760933.239467:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024561024:192.168.203.107@tcp:38:kworker.0 00010000:00010000:1.0:1789760933.495023:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789760933.750917:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789760934.518956:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789760934.774910:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789760935.542985:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789760935.798941:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789760936.566975:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789760936.822911:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789760937.590994:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789760937.846905:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:3.0:1789760938.358964:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789760938.358975:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789760938.358979:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789760938.358991:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 372 00000100:00080000:3.0:1789760938.359003:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789760938.359010:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:3.0:1789760938.359056:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872c380 x1876700024561152/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789760938.359087:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789760938.359091:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789760938.359094:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789760938.359098:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 372 00000100:00080000:3.0:1789760938.359122:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789760938.359128:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789760938.359138:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789760938.359150:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872df80 x1876700024561280/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789760938.359161:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1789760938.359164:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024561152:192.168.203.107@tcp:250:kworker.0 00000100:00080000:3.0:1789760938.359165:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789760938.359168:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789760938.359175:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 372 00000100:00080000:3.0:1789760938.359178:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789760938.359181:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789760938.359184:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789760938.359193:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872f800 x1876700024561408/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789760938.359268:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789760938.359322:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760938/real 1789760938] req@ffffa06c1872c380 x1876700024561152/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789760963 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789760938.359345:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872c380 x1876700024561152/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789760963 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789760938.359357:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789760938.359362:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789760938.359367:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024561152:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789760938.359397:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024561280:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789760938.359422:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789760938.359436:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760938/real 1789760938] req@ffffa06c1872df80 x1876700024561280/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760963 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789760938.359449:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872df80 x1876700024561280/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760963 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789760938.359459:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789760938.359463:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789760938.359475:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024561280:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789760938.359483:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024561408:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789760938.359507:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789760938.359520:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760938/real 1789760938] req@ffffa06c1872f800 x1876700024561408/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760963 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789760938.359532:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872f800 x1876700024561408/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760963 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789760938.359541:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789760938.359545:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789760938.359549:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024561408:192.168.203.107@tcp:38:kworker.0 00010000:00010000:2.0:1789760938.614994:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789760938.870941:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789760939.638970:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789760939.894918:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789760940.662945:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789760940.918933:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789760941.687055:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789760941.942908:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789760942.710986:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789760942.966906:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789760943.478957:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789760943.478965:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789760943.478968:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789760943.478977:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 377 00000100:00080000:2.0:1789760943.478986:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789760943.478991:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789760943.479023:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e53b80 x1876700024561536/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789760943.479045:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789760943.479048:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789760943.479050:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789760943.479052:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 377 00000100:00080000:2.0:1789760943.479069:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789760943.479073:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789760943.479081:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789760943.479094:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50380 x1876700024561664/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789760943.479095:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e53b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024561536:192.168.203.107@tcp:250:kworker.0 00000100:00080000:2.0:1789760943.479111:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789760943.479114:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789760943.479115:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789760943.479126:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 377 00000100:00080000:2.0:1789760943.479129:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789760943.479135:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789760943.479139:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789760943.479147:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51f80 x1876700024561792/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789760943.479170:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789760943.479208:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760943/real 1789760943] req@ffffa06c07e53b80 x1876700024561536/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789760968 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789760943.479227:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e53b80 x1876700024561536/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789760968 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789760943.479237:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789760943.479241:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789760943.479245:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e53b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024561536:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789760943.479277:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024561664:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789760943.479299:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789760943.479311:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760943/real 1789760943] req@ffffa06c07e50380 x1876700024561664/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760968 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789760943.479324:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50380 x1876700024561664/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760968 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789760943.479333:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789760943.479336:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789760943.479340:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024561664:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789760943.479347:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024561792:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789760943.479363:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789760943.479372:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760943/real 1789760943] req@ffffa06c07e51f80 x1876700024561792/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760968 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789760943.479383:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51f80 x1876700024561792/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760968 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789760943.479392:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789760943.479395:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789760943.479398:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024561792:192.168.203.107@tcp:38:kworker.0 00010000:00010000:0.0:1789760943.734957:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789760943.990919:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789760944.758980:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789760945.014957:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789760945.782950:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789760946.038900:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789760946.806970:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789760947.062919:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789760947.830955:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789760948.086914:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789760948.598945:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789760948.598955:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789760948.598958:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789760948.598967:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 382 00000100:00080000:2.0:1789760948.598977:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789760948.598984:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789760948.599017:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e52680 x1876700024561920/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789760948.599066:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789760948.599070:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789760948.599072:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789760948.599077:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 382 00000100:00080000:2.0:1789760948.599080:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789760948.599085:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789760948.599095:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789760948.599111:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e53800 x1876700024562048/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789760948.599113:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e52680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024561920:192.168.203.107@tcp:250:kworker.0 00000100:00080000:2.0:1789760948.599131:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789760948.599135:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789760948.599137:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789760948.599142:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 382 00000100:00080000:2.0:1789760948.599145:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789760948.599150:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789760948.599155:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789760948.599166:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51c00 x1876700024562176/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789760948.599177:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789760948.599220:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760948/real 1789760948] req@ffffa06c07e52680 x1876700024561920/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789760973 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789760948.599239:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e52680 x1876700024561920/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789760973 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789760948.599250:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789760948.599254:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789760948.599259:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e52680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024561920:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789760948.599281:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e53800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024562048:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789760948.599299:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789760948.599310:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760948/real 1789760948] req@ffffa06c07e53800 x1876700024562048/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760973 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789760948.599323:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e53800 x1876700024562048/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760973 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789760948.599332:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789760948.599335:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789760948.599339:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e53800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024562048:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789760948.599346:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024562176:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789760948.599365:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789760948.599373:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760948/real 1789760948] req@ffffa06c07e51c00 x1876700024562176/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760973 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789760948.599393:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51c00 x1876700024562176/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760973 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789760948.599398:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789760948.599400:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789760948.599402:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024562176:192.168.203.107@tcp:38:kworker.0 00010000:00010000:2.0:1789760948.854961:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789760949.110911:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789760949.879043:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789760950.134925:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789760950.903002:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789760951.158917:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789760951.926967:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789760952.182938:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789760952.950940:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789760953.206915:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789760953.718941:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789760953.718954:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789760953.718959:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789760953.718991:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 387 00000100:00080000:2.0:1789760953.719006:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789760953.719014:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789760953.719054:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e53480 x1876700024562304/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789760953.719077:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789760953.719081:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789760953.719083:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789760953.719088:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 387 00000100:00080000:2.0:1789760953.719092:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789760953.719097:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789760953.719111:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789760953.719120:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e53480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024562304:192.168.203.107@tcp:250:kworker.0 00000100:00100000:2.0:1789760953.719130:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50a80 x1876700024562432/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789760953.719145:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789760953.719148:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789760953.719150:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789760953.719153:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 387 00000100:00080000:2.0:1789760953.719156:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789760953.719160:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789760953.719165:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789760953.719183:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50700 x1876700024562560/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789760953.719233:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024562432:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789760953.719271:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024562560:192.168.203.107@tcp:38:kworker.0 00010000:00010000:0.0:1789760953.975031:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789760954.230981:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789760954.999030:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789760955.254936:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789760956.023024:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789760956.278957:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000400:00000100:2.0:1789760956.791159:0:2328:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.107:988 00000400:00000100:2.0:1789760956.791244:0:2328:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.107@tcp at host 192.168.203.107 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1789760956.791279:0:2328:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.107@tcp 00000800:00000100:2.0:1789760956.791341:0:2328:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.107@tcp 00000800:00000100:2.0:1789760956.791354:0:2328:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.107@tcp 00000100:00100000:1.0:1789760956.791433:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760953/real 1789760956] req@ffffa06c07e50a80 x1876700024562432/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760978 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789760956.791467:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50a80 x1876700024562432/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760978 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789760956.791480:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789760956.791485:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789760956.791503:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024562432:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789760956.791526:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760953/real 1789760956] req@ffffa06c07e50700 x1876700024562560/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760978 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789760956.791542:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50700 x1876700024562560/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760978 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789760956.791550:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789760956.791553:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789760956.791557:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024562560:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789760956.791564:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760953/real 1789760956] req@ffffa06c07e53480 x1876700024562304/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789760978 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789760956.791576:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e53480 x1876700024562304/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789760978 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789760956.791584:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789760956.791587:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789760956.791590:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e53480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024562304:192.168.203.107@tcp:250:kworker.0 00010000:00010000:0.0:1789760957.046960:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789760957.302903:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789760958.070983:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789760958.326925:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789760958.838961:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789760958.838974:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789760958.838978:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789760958.838988:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 392 00000100:00080000:2.0:1789760958.839002:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789760958.839012:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789760958.839061:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e53100 x1876700024562688/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789760958.839113:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789760958.839118:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789760958.839120:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789760958.839126:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 392 00000100:00080000:2.0:1789760958.839130:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789760958.839135:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789760958.839148:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789760958.839164:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e52d80 x1876700024562816/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789760958.839200:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789760958.839204:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789760958.839206:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789760958.839210:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 392 00000100:00080000:2.0:1789760958.839215:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789760958.839219:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789760958.839226:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789760958.839238:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e52a00 x1876700024562944/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789760958.839366:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e53100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024562688:192.168.203.107@tcp:250:kworker.0 00000800:00000100:1.0:1789760958.839460:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789760958.839546:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760958/real 1789760958] req@ffffa06c07e53100 x1876700024562688/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789760983 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789760958.839572:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e53100 x1876700024562688/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789760983 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789760958.839595:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789760958.839600:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789760958.839605:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e53100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024562688:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789760958.839615:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e52d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024562816:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789760958.839650:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789760958.839666:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760958/real 1789760958] req@ffffa06c07e52d80 x1876700024562816/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760983 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789760958.839681:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e52d80 x1876700024562816/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760983 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789760958.839689:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789760958.839692:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789760958.839696:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e52d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024562816:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789760958.839704:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e52a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024562944:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789760958.839731:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789760958.839744:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760958/real 1789760958] req@ffffa06c07e52a00 x1876700024562944/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760983 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789760958.839758:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e52a00 x1876700024562944/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760983 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789760958.839767:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789760958.839772:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789760958.839776:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e52a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024562944:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789760959.094979:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789760959.350989:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789760960.119059:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789760960.374916:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789760961.142973:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789760961.399025:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789760962.166967:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789760962.422950:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789760963.191020:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789760963.446928:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789760963.959007:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789760963.959019:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789760963.959023:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789760963.959033:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 397 00000100:00080000:0.0:1789760963.959045:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789760963.959053:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789760963.959104:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d1500 x1876700024563072/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789760963.959132:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789760963.959136:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789760963.959137:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789760963.959140:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 397 00000100:00080000:0.0:1789760963.959143:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789760963.959146:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789760963.959155:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789760963.959180:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d2680 x1876700024563200/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789760963.959192:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789760963.959195:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789760963.959196:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789760963.959199:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 397 00000100:00080000:0.0:1789760963.959203:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1789760963.959213:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024563072:192.168.203.107@tcp:250:kworker.0 00000100:00080000:0.0:1789760963.959228:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789760963.959243:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789760963.959263:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0380 x1876700024563328/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789760963.959326:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789760963.959376:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760963/real 1789760963] req@ffffa06c091d1500 x1876700024563072/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789760988 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789760963.959394:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d1500 x1876700024563072/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789760988 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789760963.959408:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789760963.959413:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789760963.959419:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024563072:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789760963.959453:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024563200:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789760963.959523:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789760963.959540:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760963/real 1789760963] req@ffffa06c091d2680 x1876700024563200/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760988 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789760963.959551:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d2680 x1876700024563200/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760988 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789760963.959567:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789760963.959570:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789760963.959573:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024563200:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789760963.959592:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024563328:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789760963.959611:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789760963.959619:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760963/real 1789760963] req@ffffa06c091d0380 x1876700024563328/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760988 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789760963.959629:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0380 x1876700024563328/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760988 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789760963.959635:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789760963.959637:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789760963.959639:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024563328:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789760964.215078:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789760964.470988:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789760965.238987:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789760965.494922:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789760966.263032:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789760966.518947:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789760967.286994:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789760967.542935:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789760968.310985:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789760968.566919:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:1.0:1789760969.078981:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789760969.078989:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789760969.078993:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789760969.079000:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 402 00000100:00080000:1.0:1789760969.079009:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789760969.079015:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789760969.079047:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2d500 x1876700024563456/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789760969.079076:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789760969.079080:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789760969.079097:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789760969.079103:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 402 00000100:00080000:1.0:1789760969.079109:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1789760969.079111:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024563456:192.168.203.107@tcp:250:kworker.0 00000100:00080000:1.0:1789760969.079125:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789760969.079137:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789760969.079158:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2c000 x1876700024563584/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789760969.079168:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789760969.079170:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789760969.079172:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789760969.079175:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 402 00000100:00080000:1.0:1789760969.079178:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789760969.079180:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000800:00000100:0.0:1789760969.079185:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000008:00100000:1.0:1789760969.079186:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789760969.079196:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2e300 x1876700024563712/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789760969.079222:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760969/real 1789760969] req@ffffa06c08e2d500 x1876700024563456/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789760994 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760969.079236:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2d500 x1876700024563456/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789760994 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760969.079244:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760969.079248:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789760969.079250:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024563456:192.168.203.107@tcp:250:kworker.0 00000100:00100000:0.0:1789760969.079284:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024563584:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789760969.079301:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789760969.079318:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760969/real 1789760969] req@ffffa06c08e2c000 x1876700024563584/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760994 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760969.079348:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2c000 x1876700024563584/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760994 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760969.079358:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760969.079362:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789760969.079366:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024563584:192.168.203.107@tcp:38:kworker.0 00000100:00100000:0.0:1789760969.079373:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024563712:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789760969.079445:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789760969.079490:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760969/real 1789760969] req@ffffa06c08e2e300 x1876700024563712/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760994 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760969.079503:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2e300 x1876700024563712/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760994 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760969.079512:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760969.079515:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789760969.079519:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024563712:192.168.203.107@tcp:38:kworker.0 00010000:00010000:2.0:1789760969.335980:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789760969.590946:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789760970.358935:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789760970.614974:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789760971.382991:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789760971.638947:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789760972.407094:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789760972.662953:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789760973.430981:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789760973.686945:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789760974.198940:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789760974.198948:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789760974.198951:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789760974.198959:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 408 00000100:00080000:2.0:1789760974.198968:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789760974.198973:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789760974.199004:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e52a00 x1876700024563840/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789760974.199027:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789760974.199029:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789760974.199031:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789760974.199034:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 408 00000100:00080000:2.0:1789760974.199054:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789760974.199058:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789760974.199066:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789760974.199083:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e52d80 x1876700024563968/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789760974.199086:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e52a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024563840:192.168.203.107@tcp:250:kworker.0 00000100:00080000:2.0:1789760974.199104:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789760974.199109:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789760974.199111:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789760974.199116:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 408 00000100:00080000:2.0:1789760974.199121:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789760974.199125:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789760974.199131:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789760974.199143:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e53100 x1876700024564096/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789760974.199145:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789760974.199190:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760974/real 1789760974] req@ffffa06c07e52a00 x1876700024563840/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789760999 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760974.199204:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e52a00 x1876700024563840/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789760999 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760974.199219:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760974.199221:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789760974.199224:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e52a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024563840:192.168.203.107@tcp:250:kworker.0 00000100:00100000:0.0:1789760974.199243:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e52d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024563968:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789760974.199259:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789760974.199267:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760974/real 1789760974] req@ffffa06c07e52d80 x1876700024563968/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760999 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760974.199275:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e52d80 x1876700024563968/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760999 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760974.199281:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760974.199283:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789760974.199285:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e52d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024563968:192.168.203.107@tcp:38:kworker.0 00000100:00100000:0.0:1789760974.199288:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e53100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024564096:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789760974.199298:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789760974.199303:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760974/real 1789760974] req@ffffa06c07e53100 x1876700024564096/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760999 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760974.199310:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e53100 x1876700024564096/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760999 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760974.199315:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760974.199317:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789760974.199319:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e53100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024564096:192.168.203.107@tcp:38:kworker.0 00010000:00010000:2.0:1789760974.454964:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789760974.710931:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789760975.479004:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789760975.734960:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789760976.502944:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789760976.758999:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789760977.526976:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789760977.782951:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789760978.550958:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789760978.806950:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789760979.318937:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789760979.318946:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789760979.318950:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789760979.318959:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 413 00000100:00080000:2.0:1789760979.318969:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789760979.318975:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789760979.319011:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e53480 x1876700024564224/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789760979.319058:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789760979.319062:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789760979.319063:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789760979.319067:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 413 00000100:00080000:2.0:1789760979.319071:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789760979.319074:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789760979.319082:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789760979.319094:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50700 x1876700024564352/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789760979.319104:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789760979.319107:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789760979.319109:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789760979.319112:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 413 00000100:00080000:2.0:1789760979.319116:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1789760979.319118:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e53480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024564224:192.168.203.107@tcp:250:kworker.0 00000100:00080000:2.0:1789760979.319122:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789760979.319132:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789760979.319140:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50a80 x1876700024564480/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789760979.319185:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789760979.319221:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760979/real 1789760979] req@ffffa06c07e53480 x1876700024564224/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761004 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760979.319237:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e53480 x1876700024564224/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761004 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760979.319246:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760979.319249:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789760979.319252:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e53480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024564224:192.168.203.107@tcp:250:kworker.0 00000100:00100000:0.0:1789760979.319258:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024564352:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789760979.319274:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789760979.319283:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760979/real 1789760979] req@ffffa06c07e50700 x1876700024564352/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761004 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760979.319292:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50700 x1876700024564352/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761004 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760979.319299:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760979.319302:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789760979.319304:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024564352:192.168.203.107@tcp:38:kworker.0 00000100:00100000:0.0:1789760979.319324:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024564480:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789760979.319373:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789760979.319384:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760979/real 1789760979] req@ffffa06c07e50a80 x1876700024564480/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761004 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760979.319414:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50a80 x1876700024564480/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761004 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760979.319425:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760979.319429:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789760979.319433:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024564480:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789760979.574952:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789760979.830962:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789760980.599033:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789760980.854923:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789760981.622995:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789760981.878961:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789760982.646993:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789760982.902929:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789760983.671017:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789760983.926997:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:3.0:1789760984.438978:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789760984.439011:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789760984.439018:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789760984.439044:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 418 00000100:00080000:3.0:1789760984.439059:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789760984.439069:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:3.0:1789760984.439139:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872d880 x1876700024564608/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789760984.439174:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789760984.439195:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789760984.439197:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789760984.439203:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 418 00000100:00080000:3.0:1789760984.439207:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789760984.439212:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789760984.439232:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789760984.439266:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872ce00 x1876700024564736/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789760984.439332:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024564608:192.168.203.107@tcp:250:kworker.0 00000100:00080000:3.0:1789760984.439427:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789760984.439433:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789760984.439436:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789760984.439441:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 418 00000800:00000100:0.0:1789760984.439450:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00080000:3.0:1789760984.439451:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789760984.439462:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789760984.439470:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789760984.439486:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872e300 x1876700024564864/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789760984.439526:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760984/real 1789760984] req@ffffa06c1872d880 x1876700024564608/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761009 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760984.439555:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872d880 x1876700024564608/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761009 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760984.439572:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760984.439577:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789760984.439584:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024564608:192.168.203.107@tcp:250:kworker.0 00000100:00100000:0.0:1789760984.439618:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024564736:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789760984.439666:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789760984.439686:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760984/real 1789760984] req@ffffa06c1872ce00 x1876700024564736/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761009 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760984.439717:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872ce00 x1876700024564736/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761009 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760984.439730:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760984.439736:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789760984.439741:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024564736:192.168.203.107@tcp:38:kworker.0 00000100:00100000:0.0:1789760984.439768:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024564864:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789760984.439801:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789760984.439819:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760984/real 1789760984] req@ffffa06c1872e300 x1876700024564864/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761009 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760984.439841:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872e300 x1876700024564864/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761009 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760984.439856:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760984.439862:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789760984.439867:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024564864:192.168.203.107@tcp:38:kworker.0 00010000:00010000:0.0:1789760984.695059:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789760984.950928:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789760985.718954:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789760985.974954:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789760986.743018:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789760986.998992:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789760987.766968:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789760988.022929:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789760988.790977:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789760989.046996:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:3.0:1789760989.558925:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789760989.558932:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789760989.558936:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789760989.558944:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 423 00000100:00080000:3.0:1789760989.558953:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789760989.558959:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:3.0:1789760989.558991:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872d180 x1876700024564992/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789760989.559013:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789760989.559015:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789760989.559017:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789760989.559019:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 423 00000100:00080000:3.0:1789760989.559022:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789760989.559025:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789760989.559047:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789760989.559048:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024564992:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789760989.559060:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872ca80 x1876700024565120/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789760989.559070:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789760989.559072:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789760989.559084:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789760989.559087:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 423 00000100:00080000:3.0:1789760989.559090:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789760989.559093:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789760989.559096:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789760989.559104:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872f100 x1876700024565248/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789760989.559149:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789760989.559183:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760989/real 1789760989] req@ffffa06c1872d180 x1876700024564992/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761014 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760989.559197:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872d180 x1876700024564992/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761014 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760989.559222:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760989.559225:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789760989.559228:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024564992:192.168.203.107@tcp:250:kworker.0 00000100:00100000:0.0:1789760989.559247:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024565120:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789760989.559266:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789760989.559294:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760989/real 1789760989] req@ffffa06c1872ca80 x1876700024565120/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761014 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760989.559318:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872ca80 x1876700024565120/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761014 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760989.559324:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760989.559327:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789760989.559329:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024565120:192.168.203.107@tcp:38:kworker.0 00000100:00100000:0.0:1789760989.559334:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024565248:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789760989.559356:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789760989.559376:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760989/real 1789760989] req@ffffa06c1872f100 x1876700024565248/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761014 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760989.559389:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872f100 x1876700024565248/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761014 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760989.559398:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760989.559400:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789760989.559403:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024565248:192.168.203.107@tcp:38:kworker.0 00010000:00010000:1.0:1789760989.815017:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789760990.070917:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789760990.839003:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789760991.094971:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789760991.863038:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789760992.118900:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789760992.886982:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789760993.142987:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789760993.910962:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789760994.166919:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789760994.678977:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789760994.678985:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789760994.678988:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789760994.679011:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 428 00000100:00080000:2.0:1789760994.679019:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789760994.679025:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789760994.679063:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51c00 x1876700024565376/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789760994.679104:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789760994.679107:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789760994.679109:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789760994.679113:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 428 00000100:00080000:2.0:1789760994.679115:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789760994.679118:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789760994.679126:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789760994.679136:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e53800 x1876700024565504/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789760994.679145:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789760994.679148:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789760994.679150:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789760994.679153:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 428 00000100:00080000:2.0:1789760994.679155:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789760994.679158:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789760994.679161:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789760994.679168:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e52680 x1876700024565632/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789760994.679184:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024565376:192.168.203.107@tcp:250:kworker.0 00000800:00000100:0.0:1789760994.679276:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789760994.679314:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760994/real 1789760994] req@ffffa06c07e51c00 x1876700024565376/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761019 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760994.679334:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51c00 x1876700024565376/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761019 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760994.679355:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760994.679360:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789760994.679364:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024565376:192.168.203.107@tcp:250:kworker.0 00000100:00100000:0.0:1789760994.679373:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e53800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024565504:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789760994.679395:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789760994.679421:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760994/real 1789760994] req@ffffa06c07e53800 x1876700024565504/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761019 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760994.679435:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e53800 x1876700024565504/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761019 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760994.679445:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760994.679448:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789760994.679452:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e53800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024565504:192.168.203.107@tcp:38:kworker.0 00000100:00100000:0.0:1789760994.679477:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e52680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024565632:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789760994.679500:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789760994.679512:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760994/real 1789760994] req@ffffa06c07e52680 x1876700024565632/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761019 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760994.679534:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e52680 x1876700024565632/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761019 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760994.679543:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760994.679546:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789760994.679550:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e52680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024565632:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789760994.935046:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789760995.190895:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789760995.958990:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789760996.215003:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789760996.983021:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789760997.238964:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789760998.006990:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789760998.262947:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789760999.031043:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789760999.287032:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:3.0:1789760999.798936:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789760999.798945:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789760999.798948:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789760999.798956:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 433 00000100:00080000:3.0:1789760999.798965:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789760999.798971:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:3.0:1789760999.799022:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872ed80 x1876700024565760/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789760999.799046:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789760999.799050:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789760999.799052:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789760999.799056:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 433 00000100:00080000:3.0:1789760999.799061:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789760999.799065:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789760999.799074:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789760999.799087:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872c700 x1876700024565888/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789760999.799097:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024565760:192.168.203.107@tcp:250:kworker.0 00000100:00080000:3.0:1789760999.799098:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789760999.799102:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789760999.799103:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789760999.799106:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 433 00000100:00080000:3.0:1789760999.799109:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789760999.799113:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789760999.799117:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789760999.799126:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872dc00 x1876700024566016/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789760999.799153:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789760999.799207:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760999/real 1789760999] req@ffffa06c1872ed80 x1876700024565760/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761024 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760999.799222:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872ed80 x1876700024565760/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761024 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760999.799229:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760999.799232:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789760999.799235:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024565760:192.168.203.107@tcp:250:kworker.0 00000100:00100000:0.0:1789760999.799253:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024565888:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789760999.799271:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789760999.799279:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760999/real 1789760999] req@ffffa06c1872c700 x1876700024565888/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761024 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760999.799287:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872c700 x1876700024565888/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761024 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760999.799302:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760999.799304:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789760999.799307:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024565888:192.168.203.107@tcp:38:kworker.0 00000100:00100000:0.0:1789760999.799311:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024566016:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789760999.799325:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789760999.799332:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760999/real 1789760999] req@ffffa06c1872dc00 x1876700024566016/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761024 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760999.799339:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872dc00 x1876700024566016/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761024 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760999.799344:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760999.799346:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789760999.799348:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024566016:192.168.203.107@tcp:38:kworker.0 00010000:00010000:1.0:1789761000.054957:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761000.310935:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789761001.079076:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761001.334938:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789761002.103022:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761002.358919:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761003.126966:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761003.382938:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789761004.151110:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761004.406905:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789761004.918930:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761004.918940:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789761004.918944:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761004.918953:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 438 00000100:00080000:2.0:1789761004.918961:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761004.918967:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789761004.918998:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51f80 x1876700024566144/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761004.919018:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761004.919020:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761004.919022:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761004.919024:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 438 00000100:00080000:2.0:1789761004.919027:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761004.919029:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789761004.919036:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761004.919044:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50380 x1876700024566272/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761004.919054:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1789761004.919058:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024566144:192.168.203.107@tcp:250:kworker.0 00000100:00080000:2.0:1789761004.919058:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761004.919060:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761004.919067:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 438 00000100:00080000:2.0:1789761004.919070:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761004.919086:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789761004.919090:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761004.919099:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e53b80 x1876700024566400/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789761004.919121:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789761004.919186:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761004/real 1789761004] req@ffffa06c07e51f80 x1876700024566144/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761029 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761004.919206:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51f80 x1876700024566144/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761029 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761004.919226:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761004.919229:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789761004.919233:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024566144:192.168.203.107@tcp:250:kworker.0 00000100:00100000:0.0:1789761004.919251:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024566272:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789761004.919270:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789761004.919279:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761004/real 1789761004] req@ffffa06c07e50380 x1876700024566272/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761029 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761004.919287:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50380 x1876700024566272/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761029 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761004.919292:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761004.919294:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789761004.919296:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024566272:192.168.203.107@tcp:38:kworker.0 00000100:00100000:0.0:1789761004.919300:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e53b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024566400:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789761004.919310:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789761004.919316:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761004/real 1789761004] req@ffffa06c07e53b80 x1876700024566400/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761029 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761004.919324:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e53b80 x1876700024566400/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761029 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761004.919329:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761004.919331:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789761004.919332:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e53b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024566400:192.168.203.107@tcp:38:kworker.0 00010000:00010000:2.0:1789761005.174982:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761005.430954:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761006.198987:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761006.454928:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789761007.223000:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761007.478949:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761008.247002:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761008.502986:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761009.271023:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761009.526947:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789761010.039016:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761010.039029:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789761010.039034:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761010.039047:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 443 00000100:00080000:2.0:1789761010.039062:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761010.039071:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789761010.039133:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50e00 x1876700024566528/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761010.039177:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761010.039181:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761010.039183:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761010.039188:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 443 00000100:00080000:2.0:1789761010.039192:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761010.039197:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789761010.039208:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761010.039226:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51500 x1876700024566656/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789761010.039237:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024566528:192.168.203.107@tcp:250:kworker.0 00000100:00080000:2.0:1789761010.039246:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761010.039279:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761010.039282:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761010.039290:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 443 00000100:00080000:2.0:1789761010.039296:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761010.039302:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789761010.039308:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761010.039337:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50000 x1876700024566784/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789761010.039353:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789761010.039425:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761010/real 1789761010] req@ffffa06c07e50e00 x1876700024566528/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761035 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761010.039449:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50e00 x1876700024566528/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761035 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761010.039463:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761010.039468:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789761010.039473:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024566528:192.168.203.107@tcp:250:kworker.0 00000100:00100000:0.0:1789761010.039521:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024566656:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789761010.039549:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789761010.039581:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761010/real 1789761010] req@ffffa06c07e51500 x1876700024566656/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761035 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761010.039592:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51500 x1876700024566656/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761035 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761010.039598:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761010.039600:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789761010.039603:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024566656:192.168.203.107@tcp:38:kworker.0 00000100:00100000:0.0:1789761010.039607:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024566784:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789761010.039622:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789761010.039650:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761010/real 1789761010] req@ffffa06c07e50000 x1876700024566784/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761035 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761010.039664:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50000 x1876700024566784/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761035 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761010.039673:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761010.039677:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789761010.039681:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024566784:192.168.203.107@tcp:38:kworker.0 00010000:00010000:0.0:1789761010.295002:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761010.550948:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761011.318992:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761011.574908:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761012.343005:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761012.598955:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761013.366963:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761013.622927:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761014.391018:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761014.646947:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789761015.158929:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761015.158937:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789761015.158941:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761015.158949:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 448 00000100:00080000:2.0:1789761015.158957:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761015.158963:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789761015.158994:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51880 x1876700024566912/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761015.159016:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761015.159020:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761015.159021:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761015.159024:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 448 00000100:00080000:2.0:1789761015.159026:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761015.159028:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789761015.159048:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761015.159059:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51180 x1876700024567040/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761015.159069:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1789761015.159123:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024566912:192.168.203.107@tcp:250:kworker.0 00000100:00080000:2.0:1789761015.159167:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761015.159171:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761015.159177:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 448 00000100:00080000:2.0:1789761015.159181:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761015.159187:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789761015.159191:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761015.159201:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e52300 x1876700024567168/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789761015.159224:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789761015.159281:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761015/real 1789761015] req@ffffa06c07e51880 x1876700024566912/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761040 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761015.159304:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51880 x1876700024566912/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761040 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761015.159316:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761015.159321:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789761015.159325:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024566912:192.168.203.107@tcp:250:kworker.0 00000100:00100000:0.0:1789761015.159336:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024567040:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789761015.159363:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789761015.159378:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761015/real 1789761015] req@ffffa06c07e51180 x1876700024567040/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761040 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761015.159401:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51180 x1876700024567040/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761040 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761015.159422:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761015.159426:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789761015.159430:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024567040:192.168.203.107@tcp:38:kworker.0 00000100:00100000:0.0:1789761015.159461:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e52300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024567168:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789761015.159484:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789761015.159496:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761015/real 1789761015] req@ffffa06c07e52300 x1876700024567168/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761040 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761015.159509:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e52300 x1876700024567168/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761040 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761015.159518:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761015.159521:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789761015.159525:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e52300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024567168:192.168.203.107@tcp:38:kworker.0 00010000:00010000:1.0:1789761015.415025:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761015.670944:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761016.438964:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761016.694941:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761017.462990:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761017.719021:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761018.486963:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761018.742931:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761019.510980:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761019.766957:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789761020.278912:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761020.278922:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789761020.278927:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761020.278935:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 454 00000100:00080000:2.0:1789761020.278943:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761020.278949:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789761020.278984:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e52300 x1876700024567296/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761020.279013:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761020.279018:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761020.279020:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761020.279034:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 454 00000100:00080000:2.0:1789761020.279051:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761020.279056:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789761020.279066:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761020.279085:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e52300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024567296:192.168.203.107@tcp:250:kworker.0 00000100:00100000:2.0:1789761020.279089:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51180 x1876700024567424/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761020.279105:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761020.279109:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761020.279112:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761020.279116:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 454 00000100:00080000:2.0:1789761020.279120:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761020.279124:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789761020.279128:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761020.279136:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51880 x1876700024567552/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789761020.279199:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024567424:192.168.203.107@tcp:38:kworker.0 00000100:00100000:0.0:1789761020.279234:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024567552:192.168.203.107@tcp:38:kworker.0 00010000:00010000:2.0:1789761020.535074:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761020.790973:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761021.558965:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761021.814924:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761022.582976:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761022.838912:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000400:00000100:3.0:1789761023.351038:0:2329:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.107:988 00000400:00000100:3.0:1789761023.351079:0:2329:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.107@tcp at host 192.168.203.107 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1789761023.351097:0:2329:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.107@tcp 00000800:00000100:3.0:1789761023.351150:0:2329:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.107@tcp 00000800:00000100:3.0:1789761023.351163:0:2329:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.107@tcp 00000100:00100000:0.0:1789761023.351329:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761020/real 1789761023] req@ffffa06c07e51180 x1876700024567424/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761045 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761023.351355:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51180 x1876700024567424/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761045 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761023.351380:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761023.351386:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789761023.351391:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024567424:192.168.203.107@tcp:38:kworker.0 00000100:00100000:0.0:1789761023.351400:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761020/real 1789761023] req@ffffa06c07e51880 x1876700024567552/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761045 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761023.351415:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51880 x1876700024567552/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761045 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761023.351424:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761023.351428:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789761023.351432:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024567552:192.168.203.107@tcp:38:kworker.0 00000100:00100000:0.0:1789761023.351438:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761020/real 1789761023] req@ffffa06c07e52300 x1876700024567296/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761045 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761023.351451:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e52300 x1876700024567296/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761045 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761023.351459:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761023.351462:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789761023.351466:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e52300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024567296:192.168.203.107@tcp:250:kworker.0 00010000:00010000:2.0:1789761023.606982:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761023.862913:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761024.630985:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761024.886944:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789761025.398959:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761025.398969:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761025.398974:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761025.398984:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 459 00000100:00080000:0.0:1789761025.399032:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761025.399038:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789761025.399066:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d3100 x1876700024567680/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761025.399100:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761025.399105:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761025.399107:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761025.399112:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 459 00000100:00080000:0.0:1789761025.399116:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761025.399121:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789761025.399131:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761025.399146:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024567680:192.168.203.107@tcp:250:kworker.0 00000100:00100000:0.0:1789761025.399148:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0a80 x1876700024567808/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761025.399167:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761025.399170:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761025.399172:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761025.399176:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 459 00000100:00080000:0.0:1789761025.399180:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761025.399184:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789761025.399189:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761025.399201:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d1f80 x1876700024567936/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789761025.399218:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761025.399265:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761025/real 1789761025] req@ffffa06c091d3100 x1876700024567680/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761050 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761025.399283:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d3100 x1876700024567680/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761050 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761025.399294:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761025.399298:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761025.399302:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024567680:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789761025.399329:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024567808:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761025.399354:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761025.399368:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761025/real 1789761025] req@ffffa06c091d0a80 x1876700024567808/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761050 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761025.399380:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0a80 x1876700024567808/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761050 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761025.399389:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761025.399392:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761025.399395:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024567808:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789761025.399401:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024567936:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761025.399420:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761025.399431:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761025/real 1789761025] req@ffffa06c091d1f80 x1876700024567936/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761050 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761025.399443:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d1f80 x1876700024567936/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761050 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761025.399465:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761025.399470:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761025.399473:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024567936:192.168.203.107@tcp:38:kworker.0 00010000:00010000:2.0:1789761025.654974:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761025.910986:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761026.679009:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761026.934969:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761027.703002:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761027.958965:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761028.727002:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761028.982937:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761029.751049:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761030.007007:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:3.0:1789761030.518935:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761030.518943:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789761030.518946:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761030.518954:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 464 00000100:00080000:3.0:1789761030.518974:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761030.518981:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:3.0:1789761030.519014:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872ea00 x1876700024568064/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761030.519039:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761030.519043:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761030.519047:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761030.519050:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 464 00000100:00080000:3.0:1789761030.519053:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761030.519055:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789761030.519063:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761030.519073:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872f480 x1876700024568192/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761030.519086:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1789761030.519090:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024568064:192.168.203.107@tcp:250:kworker.0 00000100:00080000:3.0:1789761030.519090:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761030.519094:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761030.519101:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 464 00000100:00080000:3.0:1789761030.519103:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761030.519119:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789761030.519124:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761030.519135:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872e680 x1876700024568320/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789761030.519149:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761030.519189:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761030/real 1789761030] req@ffffa06c1872ea00 x1876700024568064/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761055 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761030.519202:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872ea00 x1876700024568064/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761055 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761030.519211:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761030.519215:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761030.519218:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024568064:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789761030.519236:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024568192:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761030.519250:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761030.519258:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761030/real 1789761030] req@ffffa06c1872f480 x1876700024568192/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761055 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761030.519277:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872f480 x1876700024568192/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761055 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761030.519283:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761030.519285:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761030.519287:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024568192:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789761030.519291:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024568320:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761030.519304:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761030.519310:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761030/real 1789761030] req@ffffa06c1872e680 x1876700024568320/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761055 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761030.519317:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872e680 x1876700024568320/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761055 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761030.519322:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761030.519323:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761030.519325:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024568320:192.168.203.107@tcp:38:kworker.0 00010000:00010000:2.0:1789761030.775004:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761031.030917:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761031.798990:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761032.054980:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761032.822969:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761033.078956:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761033.846975:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761034.102990:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761034.870989:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761035.126944:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789761035.638936:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761035.638945:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789761035.638948:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761035.638973:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 469 00000100:00080000:2.0:1789761035.638983:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761035.638990:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789761035.639026:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50000 x1876700024568448/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761035.639053:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761035.639059:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761035.639061:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761035.639066:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 469 00000100:00080000:2.0:1789761035.639071:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761035.639077:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789761035.639088:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761035.639105:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51500 x1876700024568576/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789761035.639110:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024568448:192.168.203.107@tcp:250:kworker.0 00000100:00080000:2.0:1789761035.639121:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761035.639125:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761035.639128:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761035.639132:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 469 00000100:00080000:2.0:1789761035.639136:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761035.639140:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789761035.639146:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761035.639183:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50e00 x1876700024568704/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789761035.639209:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761035.639282:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761035/real 1789761035] req@ffffa06c07e50000 x1876700024568448/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761060 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761035.639296:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50000 x1876700024568448/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761060 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761035.639304:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761035.639306:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761035.639309:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024568448:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789761035.639329:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024568576:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761035.639346:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761035.639355:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761035/real 1789761035] req@ffffa06c07e51500 x1876700024568576/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761060 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761035.639363:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51500 x1876700024568576/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761060 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761035.639370:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761035.639373:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761035.639377:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024568576:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789761035.639383:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024568704:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761035.639404:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761035.639415:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761035/real 1789761035] req@ffffa06c07e50e00 x1876700024568704/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761060 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761035.639438:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50e00 x1876700024568704/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761060 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761035.639444:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761035.639446:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761035.639448:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024568704:192.168.203.107@tcp:38:kworker.0 00010000:00010000:2.0:1789761035.894992:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761036.150966:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761036.919059:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761037.175018:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761037.943055:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761038.198956:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761038.966948:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761039.222937:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761039.991042:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761040.246936:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:3.0:1789761040.758939:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761040.758948:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789761040.758951:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761040.758959:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 474 00000100:00080000:3.0:1789761040.758968:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761040.758976:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:3.0:1789761040.759013:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872fb80 x1876700024568832/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761040.759040:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761040.759043:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761040.759045:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761040.759048:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 474 00000100:00080000:3.0:1789761040.759051:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761040.759055:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789761040.759063:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761040.759077:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024568832:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789761040.759078:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872e680 x1876700024568960/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761040.759093:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761040.759096:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761040.759108:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761040.759112:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 474 00000100:00080000:3.0:1789761040.759116:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761040.759119:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789761040.759123:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761040.759135:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872f480 x1876700024569088/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789761040.759137:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761040.759173:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761040/real 1789761040] req@ffffa06c1872fb80 x1876700024568832/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761065 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761040.759187:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872fb80 x1876700024568832/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761065 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761040.759195:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761040.759198:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761040.759201:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024568832:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789761040.759218:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024568960:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761040.759233:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761040.759249:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761040/real 1789761040] req@ffffa06c1872e680 x1876700024568960/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761065 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761040.759257:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872e680 x1876700024568960/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761065 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761040.759263:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761040.759265:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761040.759268:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024568960:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789761040.759271:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024569088:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761040.759288:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761040.759298:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761040/real 1789761040] req@ffffa06c1872f480 x1876700024569088/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761065 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761040.759311:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872f480 x1876700024569088/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761065 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761040.759320:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761040.759324:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761040.759328:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024569088:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789761041.015014:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761041.271039:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761042.039006:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761042.294981:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761043.062967:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761043.318921:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761044.086994:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761044.342986:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761045.111020:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761045.366969:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789761045.879015:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761045.879026:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789761045.879031:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761045.879043:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 479 00000100:00080000:2.0:1789761045.879056:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761045.879064:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789761045.879106:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e53b80 x1876700024569216/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761045.879138:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761045.879143:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761045.879145:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761045.879150:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 479 00000100:00080000:2.0:1789761045.879155:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761045.879161:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789761045.879172:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761045.879184:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e53b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024569216:192.168.203.107@tcp:250:kworker.0 00000100:00100000:2.0:1789761045.879193:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50380 x1876700024569344/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761045.879243:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761045.879248:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761045.879251:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:1.0:1789761045.879256:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00080000:2.0:1789761045.879261:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 479 00000100:00080000:2.0:1789761045.879270:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761045.879278:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789761045.879286:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761045.879303:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51f80 x1876700024569472/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789761045.879310:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761045/real 1789761045] req@ffffa06c07e53b80 x1876700024569216/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761070 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761045.879342:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e53b80 x1876700024569216/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761070 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761045.879367:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761045.879371:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761045.879375:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e53b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024569216:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789761045.879395:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024569344:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761045.879411:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761045.879420:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761045/real 1789761045] req@ffffa06c07e50380 x1876700024569344/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761070 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761045.879428:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50380 x1876700024569344/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761070 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761045.879434:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761045.879438:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761045.879458:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024569344:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789761045.879465:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024569472:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761045.879488:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761045.879501:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761045/real 1789761045] req@ffffa06c07e51f80 x1876700024569472/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761070 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761045.879526:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51f80 x1876700024569472/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761070 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761045.879543:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761045.879546:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761045.879549:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024569472:192.168.203.107@tcp:38:kworker.0 00010000:00010000:2.0:1789761046.135089:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761046.390945:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761047.158973:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761047.414944:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761048.182989:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761048.438926:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761049.206987:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761049.462948:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761050.230948:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761050.486964:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789761050.998952:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761050.998976:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761050.998979:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761050.998987:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 484 00000100:00080000:0.0:1789761050.998996:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761050.999002:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789761050.999031:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0700 x1876700024569600/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761050.999056:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761050.999059:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761050.999060:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761050.999063:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 484 00000100:00080000:0.0:1789761050.999065:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761050.999068:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789761050.999074:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761050.999085:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d1f80 x1876700024569728/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761050.999094:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761050.999096:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761050.999098:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1789761050.999102:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024569600:192.168.203.107@tcp:250:kworker.0 00000100:00080000:0.0:1789761050.999103:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 484 00000100:00080000:0.0:1789761050.999108:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761050.999113:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789761050.999115:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761050.999122:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0a80 x1876700024569856/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789761050.999158:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761050.999195:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761050/real 1789761050] req@ffffa06c091d0700 x1876700024569600/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761075 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761050.999210:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0700 x1876700024569600/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761075 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761050.999217:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761050.999220:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761050.999222:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024569600:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789761050.999241:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024569728:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761050.999255:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761050.999263:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761050/real 1789761050] req@ffffa06c091d1f80 x1876700024569728/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761075 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761050.999271:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d1f80 x1876700024569728/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761075 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761050.999283:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761050.999285:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761050.999288:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024569728:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789761050.999292:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024569856:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761050.999305:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761050.999312:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761050/real 1789761050] req@ffffa06c091d0a80 x1876700024569856/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761075 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761050.999319:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0a80 x1876700024569856/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761075 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761050.999324:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761050.999326:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761050.999328:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024569856:192.168.203.107@tcp:38:kworker.0 00010000:00010000:0.0:1789761051.255001:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761051.510940:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761052.278947:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761052.534914:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761053.303041:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761053.559007:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761054.326974:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761054.582922:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761055.350940:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761055.606936:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:1.0:1789761056.118958:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761056.118967:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789761056.118970:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761056.118978:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 489 00000100:00080000:1.0:1789761056.118986:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761056.119007:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789761056.119035:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2c700 x1876700024569984/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761056.119057:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761056.119060:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761056.119063:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761056.119068:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 489 00000100:00080000:1.0:1789761056.119072:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761056.119077:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789761056.119086:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761056.119091:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024569984:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789761056.119104:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2ca80 x1876700024570112/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761056.119115:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761056.119118:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761056.119120:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761056.119123:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 489 00000100:00080000:1.0:1789761056.119136:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761056.119140:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000800:00000100:0.0:1789761056.119145:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000008:00100000:1.0:1789761056.119145:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761056.119165:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2ed80 x1876700024570240/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789761056.119202:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761056/real 1789761056] req@ffffa06c08e2c700 x1876700024569984/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761081 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761056.119228:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2c700 x1876700024569984/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761081 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761056.119236:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761056.119239:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789761056.119242:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024569984:192.168.203.107@tcp:250:kworker.0 00000100:00100000:0.0:1789761056.119261:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024570112:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789761056.119279:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789761056.119288:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761056/real 1789761056] req@ffffa06c08e2ca80 x1876700024570112/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761081 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761056.119296:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2ca80 x1876700024570112/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761081 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761056.119302:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761056.119304:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789761056.119306:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024570112:192.168.203.107@tcp:38:kworker.0 00000100:00100000:0.0:1789761056.119310:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024570240:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789761056.119342:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789761056.119349:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761056/real 1789761056] req@ffffa06c08e2ed80 x1876700024570240/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761081 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761056.119356:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2ed80 x1876700024570240/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761081 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761056.119363:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761056.119366:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789761056.119370:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024570240:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789761056.374967:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761056.630931:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761057.398966:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761057.654919:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761058.422945:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761058.678929:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789761059.446999:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761059.702956:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761060.470957:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761060.726945:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:3.0:1789761061.238932:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761061.238940:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789761061.238944:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761061.238952:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 495 00000100:00080000:3.0:1789761061.238961:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761061.238967:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:3.0:1789761061.239000:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872ea00 x1876700024570368/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761061.239036:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761061.239045:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761061.239046:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761061.239051:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 495 00000100:00080000:3.0:1789761061.239056:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761061.239078:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789761061.239079:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024570368:192.168.203.107@tcp:250:kworker.0 00000008:00100000:3.0:1789761061.239088:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761061.239101:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872dc00 x1876700024570496/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761061.239111:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761061.239114:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761061.239116:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761061.239118:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 495 00000100:00080000:3.0:1789761061.239122:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761061.239133:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000800:00000100:0.0:1789761061.239134:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000008:00100000:3.0:1789761061.239137:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761061.239147:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872c700 x1876700024570624/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789761061.239172:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761061/real 1789761061] req@ffffa06c1872ea00 x1876700024570368/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761086 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761061.239187:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872ea00 x1876700024570368/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761086 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761061.239195:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761061.239198:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789761061.239200:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024570368:192.168.203.107@tcp:250:kworker.0 00000100:00100000:0.0:1789761061.239217:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024570496:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789761061.239231:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789761061.239238:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761061/real 1789761061] req@ffffa06c1872dc00 x1876700024570496/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761086 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761061.239257:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872dc00 x1876700024570496/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761086 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761061.239263:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761061.239265:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789761061.239268:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024570496:192.168.203.107@tcp:38:kworker.0 00000100:00100000:0.0:1789761061.239271:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024570624:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789761061.239284:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789761061.239290:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761061/real 1789761061] req@ffffa06c1872c700 x1876700024570624/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761086 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761061.239297:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872c700 x1876700024570624/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761086 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761061.239302:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761061.239303:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789761061.239305:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024570624:192.168.203.107@tcp:38:kworker.0 00010000:00010000:1.0:1789761061.494997:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761061.750914:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789761062.518958:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761062.774915:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761063.542964:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761063.798915:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761064.566945:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761064.822901:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789761065.590980:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761065.846914:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789761066.358997:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761066.359007:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789761066.359012:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761066.359022:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 500 00000100:00080000:2.0:1789761066.359032:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761066.359040:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789761066.359078:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e52680 x1876700024570752/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761066.359107:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761066.359125:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761066.359127:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761066.359133:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 500 00000100:00080000:2.0:1789761066.359140:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761066.359144:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789761066.359155:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761066.359169:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e52680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024570752:192.168.203.107@tcp:250:kworker.0 00000100:00100000:2.0:1789761066.359173:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e53800 x1876700024570880/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761066.359194:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761066.359198:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761066.359200:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761066.359204:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 500 00000100:00080000:2.0:1789761066.359208:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761066.359212:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789761066.359219:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761066.359232:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51c00 x1876700024571008/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789761066.359252:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789761066.359303:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761066/real 1789761066] req@ffffa06c07e52680 x1876700024570752/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761091 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761066.359318:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e52680 x1876700024570752/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761091 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761066.359338:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761066.359342:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789761066.359345:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e52680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024570752:192.168.203.107@tcp:250:kworker.0 00000100:00100000:0.0:1789761066.359371:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e53800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024570880:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789761066.359395:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789761066.359409:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761066/real 1789761066] req@ffffa06c07e53800 x1876700024570880/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761091 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761066.359423:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e53800 x1876700024570880/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761091 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761066.359432:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761066.359437:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789761066.359440:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e53800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024570880:192.168.203.107@tcp:38:kworker.0 00000100:00100000:0.0:1789761066.359448:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024571008:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789761066.359470:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789761066.359482:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761066/real 1789761066] req@ffffa06c07e51c00 x1876700024571008/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761091 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761066.359494:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51c00 x1876700024571008/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761091 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761066.359503:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761066.359507:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789761066.359510:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024571008:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789761066.615002:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761066.870949:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761067.639057:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761067.894955:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789761068.663008:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761068.919023:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761069.687022:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761069.942908:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761070.710960:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761070.966945:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:3.0:1789761071.478966:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761071.478974:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789761071.478977:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761071.478985:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 505 00000100:00080000:3.0:1789761071.478994:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761071.478999:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:3.0:1789761071.479041:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872ed80 x1876700024571136/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761071.479093:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761071.479098:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761071.479100:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761071.479105:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 505 00000100:00080000:3.0:1789761071.479110:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761071.479115:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789761071.479129:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761071.479146:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872f100 x1876700024571264/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761071.479156:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761071.479159:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761071.479161:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761071.479163:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 505 00000100:00080000:3.0:1789761071.479166:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761071.479170:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789761071.479185:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761071.479188:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024571136:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789761071.479195:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872ca80 x1876700024571392/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789761071.479260:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789761071.479293:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761071/real 1789761071] req@ffffa06c1872ed80 x1876700024571136/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761096 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761071.479318:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872ed80 x1876700024571136/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761096 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761071.479326:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761071.479329:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789761071.479332:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024571136:192.168.203.107@tcp:250:kworker.0 00000100:00100000:0.0:1789761071.479337:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024571264:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789761071.479350:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789761071.479358:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761071/real 1789761071] req@ffffa06c1872f100 x1876700024571264/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761096 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761071.479365:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872f100 x1876700024571264/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761096 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761071.479371:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761071.479372:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789761071.479374:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024571264:192.168.203.107@tcp:38:kworker.0 00000100:00100000:0.0:1789761071.479391:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024571392:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789761071.479401:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789761071.479417:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761071/real 1789761071] req@ffffa06c1872ca80 x1876700024571392/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761096 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761071.479424:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872ca80 x1876700024571392/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761096 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761071.479429:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761071.479431:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789761071.479434:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024571392:192.168.203.107@tcp:38:kworker.0 00010000:00010000:1.0:1789761071.734983:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761071.990933:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761072.759009:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761073.014992:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761073.783042:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761074.038903:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761074.806949:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761075.062925:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761075.831015:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761076.086915:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789761076.598961:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761076.598972:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761076.598975:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761076.598985:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 510 00000100:00080000:0.0:1789761076.598995:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761076.599002:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789761076.599036:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d3100 x1876700024571520/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761076.599064:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761076.599068:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761076.599070:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761076.599074:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 510 00000100:00080000:0.0:1789761076.599079:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761076.599082:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789761076.599091:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761076.599104:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0380 x1876700024571648/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789761076.599112:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024571520:192.168.203.107@tcp:250:kworker.0 00000100:00080000:0.0:1789761076.599133:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761076.599137:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761076.599139:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761076.599144:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 510 00000100:00080000:0.0:1789761076.599158:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761076.599163:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000800:00000100:1.0:1789761076.599168:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000008:00100000:0.0:1789761076.599173:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761076.599189:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d2680 x1876700024571776/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789761076.599208:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761076/real 1789761076] req@ffffa06c091d3100 x1876700024571520/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761101 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761076.599229:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d3100 x1876700024571520/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761101 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761076.599237:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761076.599240:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761076.599243:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024571520:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789761076.599270:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024571648:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761076.599287:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761076.599296:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761076/real 1789761076] req@ffffa06c091d0380 x1876700024571648/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761101 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761076.599304:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0380 x1876700024571648/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761101 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761076.599310:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761076.599312:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761076.599314:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024571648:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789761076.599318:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024571776:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761076.599329:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761076.599334:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761076/real 1789761076] req@ffffa06c091d2680 x1876700024571776/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761101 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761076.599342:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d2680 x1876700024571776/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761101 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761076.599360:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761076.599364:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761076.599368:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024571776:192.168.203.107@tcp:38:kworker.0 00010000:00010000:1.0:1789761076.854981:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761077.110904:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761077.879011:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761078.134919:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761078.902976:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761079.158935:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761079.926965:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761080.182898:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761080.950984:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761081.206968:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789761081.718962:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761081.718973:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761081.718978:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761081.718988:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 515 00000100:00080000:0.0:1789761081.718999:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761081.719006:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789761081.719043:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d1500 x1876700024571904/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761081.719073:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761081.719077:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761081.719079:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761081.719095:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 515 00000100:00080000:0.0:1789761081.719099:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761081.719103:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789761081.719127:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761081.719141:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d2a00 x1876700024572032/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789761081.719159:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024571904:192.168.203.107@tcp:250:kworker.0 00000100:00080000:0.0:1789761081.719176:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761081.719181:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761081.719183:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761081.719188:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 515 00000100:00080000:0.0:1789761081.719194:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761081.719199:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789761081.719205:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761081.719218:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0e00 x1876700024572160/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789761081.719225:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761081.719275:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761081/real 1789761081] req@ffffa06c091d1500 x1876700024571904/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761106 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761081.719289:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d1500 x1876700024571904/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761106 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761081.719297:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761081.719302:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761081.719306:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024571904:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789761081.719330:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024572032:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761081.719355:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761081.719379:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761081/real 1789761081] req@ffffa06c091d2a00 x1876700024572032/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761106 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761081.719393:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d2a00 x1876700024572032/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761106 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761081.719401:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761081.719403:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761081.719406:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024572032:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789761081.719411:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024572160:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761081.719427:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761081.719434:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761081/real 1789761081] req@ffffa06c091d0e00 x1876700024572160/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761106 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761081.719442:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0e00 x1876700024572160/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761106 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761081.719447:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761081.719449:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761081.719451:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024572160:192.168.203.107@tcp:38:kworker.0 00010000:00010000:1.0:1789761081.974963:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761082.230906:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761082.998984:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761083.254909:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761084.023008:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761084.278974:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761085.046949:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761085.302933:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761086.071049:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761086.326979:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:3.0:1789761086.838960:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761086.838971:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789761086.838976:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761086.838996:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 520 00000100:00080000:3.0:1789761086.839014:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761086.839020:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:3.0:1789761086.839048:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872d180 x1876700024572288/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761086.839075:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761086.839078:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761086.839079:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761086.839082:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 520 00000100:00080000:3.0:1789761086.839085:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761086.839087:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789761086.839099:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761086.839108:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872e300 x1876700024572416/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761086.839117:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761086.839120:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761086.839121:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761086.839124:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 520 00000100:00080000:3.0:1789761086.839127:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761086.839130:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789761086.839133:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761086.839147:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872ce00 x1876700024572544/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789761086.839151:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024572288:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789761086.839311:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024572416:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789761086.839373:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024572544:192.168.203.107@tcp:38:kworker.0 00010000:00010000:1.0:1789761087.095056:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761087.350945:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761088.119100:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761088.374942:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761089.142973:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761089.398947:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000400:00000100:2.0:1789761089.911121:0:2330:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.107:988 00000400:00000100:2.0:1789761089.911167:0:2330:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.107@tcp at host 192.168.203.107 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1789761089.911181:0:2330:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.107@tcp 00000800:00000100:2.0:1789761089.911210:0:2330:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.107@tcp 00000800:00000100:2.0:1789761089.911216:0:2330:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.107@tcp 00000100:00100000:1.0:1789761089.911246:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761086/real 1789761089] req@ffffa06c1872ce00 x1876700024572544/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761111 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761089.911267:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872ce00 x1876700024572544/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761111 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761089.911276:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761089.911280:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761089.911283:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024572544:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789761089.911288:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761086/real 1789761089] req@ffffa06c1872d180 x1876700024572288/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761111 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761089.911297:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872d180 x1876700024572288/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761111 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761089.911301:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761089.911304:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761089.911322:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024572288:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789761089.911330:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761086/real 1789761089] req@ffffa06c1872e300 x1876700024572416/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761111 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761089.911345:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872e300 x1876700024572416/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761111 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761089.911354:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761089.911357:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761089.911361:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024572416:192.168.203.107@tcp:38:kworker.0 00010000:00010000:1.0:1789761090.167059:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761090.422974:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789761091.190980:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761091.446940:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:1.0:1789761091.958939:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761091.958947:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789761091.958951:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761091.958966:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 525 00000100:00080000:1.0:1789761091.958980:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761091.958987:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789761091.959020:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2d180 x1876700024572672/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761091.959051:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761091.959055:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761091.959057:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761091.959062:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 525 00000100:00080000:1.0:1789761091.959066:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761091.959083:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789761091.959105:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761091.959113:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024572672:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789761091.959124:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2ed80 x1876700024572800/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761091.959134:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761091.959141:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761091.959143:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761091.959146:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 525 00000100:00080000:1.0:1789761091.959149:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761091.959153:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789761091.959157:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761091.959165:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2ca80 x1876700024572928/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789761091.959181:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789761091.959222:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761091/real 1789761091] req@ffffa06c08e2d180 x1876700024572672/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761116 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761091.959241:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2d180 x1876700024572672/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761116 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761091.959252:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761091.959256:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789761091.959260:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024572672:192.168.203.107@tcp:250:kworker.0 00000100:00100000:0.0:1789761091.959286:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024572800:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789761091.959318:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789761091.959331:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761091/real 1789761091] req@ffffa06c08e2ed80 x1876700024572800/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761116 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761091.959344:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2ed80 x1876700024572800/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761116 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761091.959352:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761091.959356:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789761091.959359:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024572800:192.168.203.107@tcp:38:kworker.0 00000100:00100000:0.0:1789761091.959366:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024572928:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789761091.959387:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789761091.959399:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761091/real 1789761091] req@ffffa06c08e2ca80 x1876700024572928/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761116 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761091.959412:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2ca80 x1876700024572928/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761116 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761091.959421:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761091.959424:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789761091.959428:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024572928:192.168.203.107@tcp:38:kworker.0 00010000:00010000:2.0:1789761092.214972:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761092.470948:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761093.238963:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761093.494920:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789761094.262957:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761094.518933:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789761095.286973:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761095.542912:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761096.310968:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761096.566922:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:3.0:1789761097.078954:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761097.078965:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789761097.078969:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761097.078980:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 530 00000100:00080000:3.0:1789761097.078993:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761097.079001:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:3.0:1789761097.079047:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872d880 x1876700024573056/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761097.079070:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761097.079073:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761097.079075:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761097.079078:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 530 00000100:00080000:3.0:1789761097.079081:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761097.079083:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789761097.079092:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761097.079109:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872f800 x1876700024573184/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789761097.079118:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024573056:192.168.203.107@tcp:250:kworker.0 00000100:00080000:3.0:1789761097.079129:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761097.079135:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761097.079137:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761097.079149:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 530 00000100:00080000:3.0:1789761097.079154:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761097.079158:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789761097.079163:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1789761097.079197:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789761097.079201:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872df80 x1876700024573312/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789761097.079242:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761097/real 1789761097] req@ffffa06c1872d880 x1876700024573056/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761122 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761097.079266:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872d880 x1876700024573056/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761122 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761097.079275:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761097.079278:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789761097.079281:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024573056:192.168.203.107@tcp:250:kworker.0 00000100:00100000:0.0:1789761097.079303:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024573184:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789761097.079321:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789761097.079329:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761097/real 1789761097] req@ffffa06c1872f800 x1876700024573184/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761122 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761097.079338:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872f800 x1876700024573184/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761122 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761097.079344:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761097.079346:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789761097.079348:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024573184:192.168.203.107@tcp:38:kworker.0 00000100:00100000:0.0:1789761097.079352:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024573312:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789761097.079362:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789761097.079368:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761097/real 1789761097] req@ffffa06c1872df80 x1876700024573312/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761122 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761097.079375:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872df80 x1876700024573312/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761122 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761097.079390:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761097.079392:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789761097.079395:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024573312:192.168.203.107@tcp:38:kworker.0 00010000:00010000:2.0:1789761097.335021:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761097.590957:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789761098.358992:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761098.615002:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761099.382987:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761099.638922:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761100.406967:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761100.662954:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789761101.431063:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761101.686914:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:3.0:1789761102.198906:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761102.198925:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789761102.198929:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761102.198937:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 536 00000100:00080000:3.0:1789761102.198946:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761102.198951:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:3.0:1789761102.198999:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872c380 x1876700024573440/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761102.199022:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761102.199025:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761102.199026:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761102.199029:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 536 00000100:00080000:3.0:1789761102.199031:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761102.199033:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789761102.199040:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761102.199052:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872df80 x1876700024573568/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761102.199060:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761102.199062:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761102.199064:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761102.199069:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 536 00000100:00100000:0.0:1789761102.199072:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024573440:192.168.203.107@tcp:250:kworker.0 00000100:00080000:3.0:1789761102.199085:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761102.199089:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789761102.199094:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761102.199105:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872f800 x1876700024573696/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789761102.199128:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789761102.199160:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761102/real 1789761102] req@ffffa06c1872c380 x1876700024573440/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761127 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761102.199174:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872c380 x1876700024573440/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761127 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761102.199181:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761102.199184:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789761102.199187:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024573440:192.168.203.107@tcp:250:kworker.0 00000100:00100000:0.0:1789761102.199193:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024573568:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789761102.199210:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789761102.199219:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761102/real 1789761102] req@ffffa06c1872df80 x1876700024573568/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761127 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761102.199226:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872df80 x1876700024573568/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761127 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761102.199231:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761102.199240:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789761102.199243:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024573568:192.168.203.107@tcp:38:kworker.0 00000100:00100000:0.0:1789761102.199262:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024573696:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789761102.199275:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789761102.199283:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761102/real 1789761102] req@ffffa06c1872f800 x1876700024573696/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761127 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761102.199290:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872f800 x1876700024573696/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761127 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761102.199296:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761102.199298:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789761102.199300:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024573696:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789761102.454956:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761102.710954:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761103.479050:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761103.734907:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761104.502998:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761104.758947:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761105.527044:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761105.782990:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761106.550967:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761106.806993:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789761107.318957:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761107.318969:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761107.318981:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761107.318994:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 541 00000100:00080000:0.0:1789761107.319008:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761107.319017:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789761107.319067:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d1180 x1876700024573824/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761107.319102:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761107.319106:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761107.319108:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761107.319113:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 541 00000100:00080000:0.0:1789761107.319134:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761107.319141:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789761107.319153:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761107.319171:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d3b80 x1876700024573952/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789761107.319177:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024573824:192.168.203.107@tcp:250:kworker.0 00000100:00080000:0.0:1789761107.319200:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761107.319205:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761107.319207:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761107.319213:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 541 00000100:00080000:0.0:1789761107.319218:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761107.319223:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789761107.319231:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761107.319247:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d2300 x1876700024574080/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789761107.319265:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761107.319314:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761107/real 1789761107] req@ffffa06c091d1180 x1876700024573824/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761132 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761107.319332:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d1180 x1876700024573824/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761132 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761107.319353:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761107.319356:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761107.319360:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024573824:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789761107.319380:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024573952:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761107.319414:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761107.319430:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761107/real 1789761107] req@ffffa06c091d3b80 x1876700024573952/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761132 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761107.319439:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d3b80 x1876700024573952/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761132 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761107.319445:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761107.319447:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761107.319449:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024573952:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789761107.319454:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024574080:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761107.319467:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761107.319473:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761107/real 1789761107] req@ffffa06c091d2300 x1876700024574080/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761132 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761107.319481:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d2300 x1876700024574080/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761132 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761107.319486:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761107.319489:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761107.319493:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024574080:192.168.203.107@tcp:38:kworker.0 00010000:00010000:2.0:1789761107.575157:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761107.830963:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761108.598928:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761108.854934:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761109.623060:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761109.878938:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761110.647058:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761110.902930:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761111.670973:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761111.926926:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789761112.438915:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761112.438923:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761112.438926:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761112.438935:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 546 00000100:00080000:0.0:1789761112.438944:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761112.438954:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789761112.438983:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0000 x1876700024574208/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761112.439004:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761112.439021:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761112.439023:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761112.439027:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 546 00000100:00080000:0.0:1789761112.439031:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761112.439034:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789761112.439041:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024574208:192.168.203.107@tcp:250:kworker.0 00000008:00100000:0.0:1789761112.439043:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761112.439063:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d2300 x1876700024574336/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761112.439077:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761112.439082:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761112.439084:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761112.439088:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 546 00000100:00080000:0.0:1789761112.439093:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1789761112.439097:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00080000:0.0:1789761112.439101:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789761112.439109:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761112.439122:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d3b80 x1876700024574464/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789761112.439139:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761112/real 1789761112] req@ffffa06c091d0000 x1876700024574208/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761137 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761112.439156:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0000 x1876700024574208/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761137 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761112.439167:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761112.439172:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761112.439176:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024574208:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789761112.439200:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024574336:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761112.439221:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761112.439231:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761112/real 1789761112] req@ffffa06c091d2300 x1876700024574336/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761137 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761112.439240:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d2300 x1876700024574336/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761137 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761112.439246:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761112.439248:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761112.439250:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024574336:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789761112.439254:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024574464:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761112.439265:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761112.439271:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761112/real 1789761112] req@ffffa06c091d3b80 x1876700024574464/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761137 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761112.439286:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d3b80 x1876700024574464/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761137 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761112.439292:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761112.439294:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761112.439296:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024574464:192.168.203.107@tcp:38:kworker.0 00010000:00010000:0.0:1789761112.694981:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761112.950909:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761113.718986:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761113.974926:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761114.742952:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761114.998898:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789761115.766957:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761116.022993:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761116.790975:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761117.046917:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789761117.558961:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761117.558969:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789761117.558972:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761117.558980:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 551 00000100:00080000:2.0:1789761117.559005:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761117.559011:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789761117.559040:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50a80 x1876700024574592/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761117.559063:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761117.559066:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761117.559067:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761117.559070:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 551 00000100:00080000:2.0:1789761117.559072:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761117.559075:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789761117.559082:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761117.559092:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50700 x1876700024574720/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761117.559102:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761117.559105:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761117.559106:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761117.559108:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 551 00000100:00080000:2.0:1789761117.559111:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1789761117.559120:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024574592:192.168.203.107@tcp:250:kworker.0 00000100:00080000:2.0:1789761117.559121:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789761117.559135:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761117.559144:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e53480 x1876700024574848/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789761117.559206:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761117.559259:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761117/real 1789761117] req@ffffa06c07e50a80 x1876700024574592/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761142 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761117.559282:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50a80 x1876700024574592/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761142 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761117.559291:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761117.559295:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761117.559299:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024574592:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789761117.559343:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024574720:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761117.559378:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761117.559393:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761117/real 1789761117] req@ffffa06c07e50700 x1876700024574720/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761142 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761117.559402:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50700 x1876700024574720/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761142 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761117.559425:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761117.559427:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761117.559430:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024574720:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789761117.559435:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e53480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024574848:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761117.559452:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761117.559460:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761117/real 1789761117] req@ffffa06c07e53480 x1876700024574848/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761142 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761117.559468:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e53480 x1876700024574848/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761142 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761117.559473:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761117.559476:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761117.559478:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e53480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024574848:192.168.203.107@tcp:38:kworker.0 00010000:00010000:2.0:1789761117.814962:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761118.070979:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789761118.839005:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761119.094942:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761119.862966:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761120.118921:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761120.887109:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761121.142924:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789761121.911035:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761122.166978:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:3.0:1789761122.679009:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761122.679024:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789761122.679030:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761122.679044:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 556 00000100:00080000:3.0:1789761122.679059:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761122.679069:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:3.0:1789761122.679120:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872d880 x1876700024574976/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761122.679148:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761122.679151:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761122.679153:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761122.679156:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 556 00000100:00080000:3.0:1789761122.679160:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761122.679162:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789761122.679191:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761122.679204:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872e300 x1876700024575104/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789761122.679213:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024574976:192.168.203.107@tcp:250:kworker.0 00000100:00080000:3.0:1789761122.679218:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761122.679223:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761122.679224:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761122.679228:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 556 00000100:00080000:3.0:1789761122.679231:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761122.679233:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789761122.679238:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761122.679247:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872d180 x1876700024575232/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789761122.679312:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761122.679364:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761122/real 1789761122] req@ffffa06c1872d880 x1876700024574976/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761147 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761122.679387:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872d880 x1876700024574976/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761147 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761122.679400:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761122.679405:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761122.679419:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024574976:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789761122.679447:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024575104:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761122.679473:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761122.679487:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761122/real 1789761122] req@ffffa06c1872e300 x1876700024575104/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761147 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761122.679500:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872e300 x1876700024575104/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761147 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761122.679510:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761122.679514:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761122.679517:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024575104:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789761122.679523:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024575232:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761122.679540:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761122.679549:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761122/real 1789761122] req@ffffa06c1872d180 x1876700024575232/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761147 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761122.679561:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872d180 x1876700024575232/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761147 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761122.679570:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761122.679573:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761122.679577:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024575232:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789761122.935012:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761123.190974:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761123.959100:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761124.214948:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789761124.982981:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761125.238981:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761126.007352:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761126.262958:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761127.031008:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761127.286987:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789761127.798965:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761127.798976:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761127.798996:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761127.799006:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 561 00000100:00080000:0.0:1789761127.799016:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761127.799029:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789761127.799061:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d1180 x1876700024575360/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761127.799088:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761127.799092:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761127.799094:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761127.799099:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 561 00000100:00080000:0.0:1789761127.799102:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761127.799106:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789761127.799115:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761127.799127:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0e00 x1876700024575488/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761127.799139:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761127.799144:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761127.799146:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761127.799149:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 561 00000100:00080000:0.0:1789761127.799152:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761127.799157:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789761127.799163:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761127.799173:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024575360:192.168.203.107@tcp:250:kworker.0 00000100:00100000:0.0:1789761127.799181:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d2a00 x1876700024575616/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789761127.799279:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761127.799354:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761127/real 1789761127] req@ffffa06c091d1180 x1876700024575360/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761152 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761127.799372:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d1180 x1876700024575360/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761152 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761127.799383:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761127.799387:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761127.799390:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024575360:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789761127.799396:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024575488:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761127.799414:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761127.799422:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761127/real 1789761127] req@ffffa06c091d0e00 x1876700024575488/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761152 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761127.799431:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0e00 x1876700024575488/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761152 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761127.799436:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761127.799438:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761127.799440:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024575488:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789761127.799463:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024575616:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761127.799475:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761127.799481:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761127/real 1789761127] req@ffffa06c091d2a00 x1876700024575616/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761152 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761127.799489:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d2a00 x1876700024575616/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761152 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761127.799504:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761127.799506:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761127.799508:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024575616:192.168.203.107@tcp:38:kworker.0 00010000:00010000:1.0:1789761128.054994:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761128.310952:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761129.078979:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761129.334955:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789761130.102950:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761130.358933:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761131.127005:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761131.382952:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761132.150965:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761132.406918:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789761132.918957:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761132.918966:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761132.918969:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761132.918977:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 566 00000100:00080000:0.0:1789761132.918987:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761132.918993:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789761132.919024:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d1500 x1876700024575744/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761132.919046:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761132.919049:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761132.919051:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761132.919053:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 566 00000100:00080000:0.0:1789761132.919056:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761132.919058:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789761132.919065:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761132.919073:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d2680 x1876700024575872/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761132.919082:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761132.919084:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761132.919087:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761132.919089:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 566 00000100:00080000:0.0:1789761132.919091:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1789761132.919104:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024575744:192.168.203.107@tcp:250:kworker.0 00000100:00080000:0.0:1789761132.919110:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789761132.919123:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761132.919135:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0380 x1876700024576000/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789761132.919192:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761132.919253:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761132/real 1789761132] req@ffffa06c091d1500 x1876700024575744/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761157 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761132.919271:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d1500 x1876700024575744/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761157 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761132.919280:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761132.919284:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761132.919287:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024575744:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789761132.919294:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024575872:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761132.919313:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761132.919322:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761132/real 1789761132] req@ffffa06c091d2680 x1876700024575872/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761157 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761132.919331:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d2680 x1876700024575872/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761157 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761132.919337:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761132.919340:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761132.919361:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024575872:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789761132.919405:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024576000:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761132.919437:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761132.919451:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761132/real 1789761132] req@ffffa06c091d0380 x1876700024576000/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761157 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761132.919462:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0380 x1876700024576000/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761157 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761132.919468:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761132.919470:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761132.919473:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024576000:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789761133.175029:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761133.430904:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789761134.199008:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761134.454908:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761135.223024:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761135.478980:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761136.246957:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761136.502929:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789761137.270970:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761137.526950:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789761138.038973:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761138.038985:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789761138.038989:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761138.038999:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 571 00000100:00080000:2.0:1789761138.039010:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761138.039019:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789761138.039054:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e53100 x1876700024576128/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761138.039096:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761138.039100:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761138.039102:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761138.039106:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 571 00000100:00080000:2.0:1789761138.039111:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761138.039116:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789761138.039125:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761138.039131:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e53100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024576128:192.168.203.107@tcp:250:kworker.0 00000100:00100000:2.0:1789761138.039143:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e52d80 x1876700024576256/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761138.039171:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761138.039175:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761138.039177:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761138.039182:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 571 00000100:00080000:2.0:1789761138.039186:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761138.039191:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789761138.039196:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1789761138.039205:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789761138.039212:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e52a00 x1876700024576384/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789761138.039254:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761138/real 1789761138] req@ffffa06c07e53100 x1876700024576128/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761163 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761138.039270:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e53100 x1876700024576128/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761163 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761138.039277:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761138.039280:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761138.039283:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e53100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024576128:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789761138.039301:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e52d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024576256:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761138.039318:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761138.039334:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761138/real 1789761138] req@ffffa06c07e52d80 x1876700024576256/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761163 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761138.039342:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e52d80 x1876700024576256/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761163 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761138.039348:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761138.039350:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761138.039353:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e52d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024576256:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789761138.039356:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e52a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024576384:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761138.039369:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761138.039374:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761138/real 1789761138] req@ffffa06c07e52a00 x1876700024576384/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761163 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761138.039382:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e52a00 x1876700024576384/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761163 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761138.039387:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761138.039389:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761138.039391:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e52a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024576384:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789761138.295027:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761138.550955:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761139.319020:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761139.574954:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761140.342987:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761140.598937:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761141.366983:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761141.622985:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761142.390984:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761142.646912:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789761143.158922:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761143.158930:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761143.158933:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761143.158941:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 576 00000100:00080000:0.0:1789761143.158950:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761143.158956:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789761143.158986:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d3100 x1876700024576512/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761143.159008:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761143.159010:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761143.159012:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761143.159015:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 576 00000100:00080000:0.0:1789761143.159017:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761143.159019:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789761143.159027:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761143.159035:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0a80 x1876700024576640/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761143.159045:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1789761143.159050:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024576512:192.168.203.107@tcp:250:kworker.0 00000100:00080000:0.0:1789761143.159059:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761143.159061:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761143.159065:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 576 00000100:00080000:0.0:1789761143.159068:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761143.159071:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789761143.159076:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761143.159093:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d1f80 x1876700024576768/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789761143.159103:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761143.159149:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761143/real 1789761143] req@ffffa06c091d3100 x1876700024576512/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761168 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761143.159184:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d3100 x1876700024576512/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761168 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761143.159196:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761143.159201:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761143.159204:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024576512:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789761143.159236:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024576640:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761143.159262:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761143.159277:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761143/real 1789761143] req@ffffa06c091d0a80 x1876700024576640/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761168 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761143.159291:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0a80 x1876700024576640/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761168 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761143.159300:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761143.159304:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761143.159307:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024576640:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789761143.159314:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024576768:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761143.159336:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761143.159349:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761143/real 1789761143] req@ffffa06c091d1f80 x1876700024576768/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761168 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761143.159361:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d1f80 x1876700024576768/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761168 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761143.159376:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761143.159379:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761143.159381:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024576768:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789761143.414947:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761143.670977:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761144.439006:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761144.694947:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761145.462983:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761145.718997:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761146.486986:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761146.742975:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761147.510970:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761147.766912:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789761148.278974:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761148.278985:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761148.278990:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761148.279001:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 582 00000100:00080000:0.0:1789761148.279014:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761148.279021:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789761148.279064:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0700 x1876700024576896/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761148.279104:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761148.279108:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761148.279109:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761148.279113:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 582 00000100:00080000:0.0:1789761148.279116:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761148.279119:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789761148.279134:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761148.279150:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d1f80 x1876700024577024/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789761148.279156:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024576896:192.168.203.107@tcp:250:kworker.0 00000100:00080000:0.0:1789761148.279178:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761148.279182:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761148.279185:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761148.279189:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 582 00000100:00080000:0.0:1789761148.279193:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761148.279197:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789761148.279203:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761148.279213:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0a80 x1876700024577152/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789761148.279229:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761148.279283:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761148/real 1789761148] req@ffffa06c091d0700 x1876700024576896/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761173 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761148.279306:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0700 x1876700024576896/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761173 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761148.279317:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761148.279322:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761148.279327:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024576896:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789761148.279357:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024577024:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761148.279384:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761148.279397:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761148/real 1789761148] req@ffffa06c091d1f80 x1876700024577024/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761173 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761148.279411:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d1f80 x1876700024577024/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761173 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761148.279435:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761148.279439:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761148.279443:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024577024:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789761148.279451:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024577152:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761148.279475:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761148.279489:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761148/real 1789761148] req@ffffa06c091d0a80 x1876700024577152/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761173 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761148.279501:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0a80 x1876700024577152/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761173 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761148.279508:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761148.279510:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761148.279512:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024577152:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789761148.535042:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761148.791002:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761149.558952:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761149.814917:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761150.582951:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761150.838973:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761151.606954:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761151.862912:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761152.630950:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761152.886912:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789761153.398957:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761153.398989:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789761153.398993:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761153.399004:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 587 00000100:00080000:2.0:1789761153.399015:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761153.399022:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789761153.399067:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e52a00 x1876700024577280/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761153.399087:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761153.399090:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761153.399091:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761153.399094:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 587 00000100:00080000:2.0:1789761153.399097:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761153.399100:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789761153.399109:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761153.399123:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e52d80 x1876700024577408/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789761153.399131:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e52a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024577280:192.168.203.107@tcp:250:kworker.0 00000100:00080000:2.0:1789761153.399139:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761153.399145:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761153.399146:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761153.399151:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 587 00000100:00080000:2.0:1789761153.399156:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761153.399160:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789761153.399166:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761153.399179:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e53100 x1876700024577536/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789761153.399235:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e52d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024577408:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789761153.399278:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e53100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024577536:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789761153.654987:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761153.910916:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761154.679030:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761154.934986:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761155.702991:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761155.959033:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000400:00000100:3.0:1789761156.471102:0:2331:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.107:988 00000400:00000100:3.0:1789761156.471139:0:2331:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.107@tcp at host 192.168.203.107 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1789761156.471155:0:2331:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.107@tcp 00000800:00000100:3.0:1789761156.471193:0:2331:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.107@tcp 00000800:00000100:3.0:1789761156.471209:0:2331:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.107@tcp 00000100:00100000:1.0:1789761156.471246:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761153/real 1789761156] req@ffffa06c07e52d80 x1876700024577408/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761178 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761156.471291:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e52d80 x1876700024577408/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761178 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761156.471306:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761156.471313:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761156.471318:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e52d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024577408:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789761156.471327:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761153/real 1789761156] req@ffffa06c07e53100 x1876700024577536/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761178 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761156.471341:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e53100 x1876700024577536/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761178 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761156.471351:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761156.471355:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761156.471376:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e53100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024577536:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789761156.471384:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761153/real 1789761156] req@ffffa06c07e52a00 x1876700024577280/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761178 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761156.471398:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e52a00 x1876700024577280/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761178 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761156.471406:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761156.471409:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761156.471438:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e52a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024577280:192.168.203.107@tcp:250:kworker.0 00010000:00010000:3.0:1789761156.726986:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761156.983004:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789761157.750977:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761158.006918:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789761158.518933:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761158.518941:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761158.518944:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761158.518952:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 592 00000100:00080000:0.0:1789761158.518961:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761158.518966:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789761158.518994:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d3100 x1876700024577664/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761158.519033:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761158.519038:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761158.519039:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761158.519043:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 592 00000100:00080000:0.0:1789761158.519046:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761158.519061:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789761158.519072:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761158.519072:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024577664:192.168.203.107@tcp:250:kworker.0 00000100:00100000:0.0:1789761158.519091:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0380 x1876700024577792/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761158.519106:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761158.519110:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761158.519113:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761158.519116:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 592 00000100:00080000:0.0:1789761158.519130:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761158.519135:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000800:00000100:1.0:1789761158.519141:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000008:00100000:0.0:1789761158.519145:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761158.519162:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d2680 x1876700024577920/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789761158.519185:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761158/real 1789761158] req@ffffa06c091d3100 x1876700024577664/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761183 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761158.519202:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d3100 x1876700024577664/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761183 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761158.519212:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761158.519216:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761158.519221:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024577664:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789761158.519245:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024577792:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761158.519269:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761158.519283:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761158/real 1789761158] req@ffffa06c091d0380 x1876700024577792/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761183 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761158.519296:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0380 x1876700024577792/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761183 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761158.519305:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761158.519309:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761158.519313:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024577792:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789761158.519319:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024577920:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761158.519340:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761158.519367:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761158/real 1789761158] req@ffffa06c091d2680 x1876700024577920/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761183 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761158.519380:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d2680 x1876700024577920/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761183 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761158.519388:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761158.519391:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761158.519394:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024577920:192.168.203.107@tcp:38:kworker.0 00010000:00010000:2.0:1789761158.774992:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761159.030937:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761159.799059:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761160.054972:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761160.823051:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761161.078977:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761161.846971:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761162.102940:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761162.870942:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761163.126950:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:1.0:1789761163.638996:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761163.639007:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789761163.639011:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761163.639022:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 597 00000100:00080000:1.0:1789761163.639031:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761163.639038:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789761163.639071:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2c700 x1876700024578048/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761163.639100:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761163.639104:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761163.639106:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761163.639112:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 597 00000100:00080000:1.0:1789761163.639117:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761163.639122:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789761163.639133:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761163.639147:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024578048:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789761163.639150:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2e300 x1876700024578176/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761163.639178:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761163.639181:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761163.639183:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761163.639186:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 597 00000100:00080000:1.0:1789761163.639190:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761163.639193:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789761163.639199:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761163.639210:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2c000 x1876700024578304/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789761163.639234:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789761163.639284:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761163/real 1789761163] req@ffffa06c08e2c700 x1876700024578048/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761188 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761163.639300:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2c700 x1876700024578048/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761188 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761163.639309:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761163.639314:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789761163.639318:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024578048:192.168.203.107@tcp:250:kworker.0 00000100:00100000:0.0:1789761163.639368:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024578176:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789761163.639394:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789761163.639405:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761163/real 1789761163] req@ffffa06c08e2e300 x1876700024578176/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761188 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761163.639414:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2e300 x1876700024578176/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761188 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761163.639420:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761163.639423:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789761163.639426:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024578176:192.168.203.107@tcp:38:kworker.0 00000100:00100000:0.0:1789761163.639430:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024578304:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789761163.639443:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789761163.639451:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761163/real 1789761163] req@ffffa06c08e2c000 x1876700024578304/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761188 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761163.639463:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2c000 x1876700024578304/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761188 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761163.639472:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761163.639476:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789761163.639479:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024578304:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789761163.895060:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761164.150959:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761164.918993:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761165.174938:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761165.943010:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761166.198997:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761166.966995:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761167.222986:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761167.990971:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761168.246982:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:1.0:1789761168.758974:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761168.758987:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789761168.758993:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761168.759005:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 602 00000100:00080000:1.0:1789761168.759019:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761168.759028:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789761168.759076:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2d500 x1876700024578432/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761168.759108:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761168.759112:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761168.759115:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761168.759120:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 602 00000100:00080000:1.0:1789761168.759123:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761168.759128:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789761168.759158:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761168.759162:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024578432:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789761168.759178:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2c000 x1876700024578560/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761168.759194:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761168.759197:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761168.759199:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761168.759203:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 602 00000100:00080000:1.0:1789761168.759208:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761168.759213:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789761168.759219:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761168.759231:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2e300 x1876700024578688/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789761168.759246:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789761168.759298:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761168/real 1789761168] req@ffffa06c08e2d500 x1876700024578432/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761193 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761168.759322:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2d500 x1876700024578432/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761193 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761168.759331:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761168.759335:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789761168.759338:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024578432:192.168.203.107@tcp:250:kworker.0 00000100:00100000:0.0:1789761168.759362:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024578560:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789761168.759381:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789761168.759390:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761168/real 1789761168] req@ffffa06c08e2c000 x1876700024578560/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761193 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761168.759398:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2c000 x1876700024578560/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761193 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761168.759404:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761168.759407:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789761168.759409:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024578560:192.168.203.107@tcp:38:kworker.0 00000100:00100000:0.0:1789761168.759413:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024578688:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789761168.759423:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789761168.759429:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761168/real 1789761168] req@ffffa06c08e2e300 x1876700024578688/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761193 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761168.759437:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2e300 x1876700024578688/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761193 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761168.759452:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761168.759455:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789761168.759457:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024578688:192.168.203.107@tcp:38:kworker.0 00010000:00010000:2.0:1789761169.015040:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761169.270967:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761170.038971:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761170.295008:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789761171.062980:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761171.318928:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761172.086964:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761172.342961:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761173.110998:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761173.366968:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789761173.878951:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761173.878959:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761173.878963:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761173.878971:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 607 00000100:00080000:0.0:1789761173.878980:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761173.878985:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789761173.879018:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d1500 x1876700024578816/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761173.879046:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761173.879049:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761173.879050:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761173.879054:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 607 00000100:00080000:0.0:1789761173.879056:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761173.879058:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789761173.879066:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761173.879075:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d2a00 x1876700024578944/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761173.879083:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1789761173.879097:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024578816:192.168.203.107@tcp:250:kworker.0 00000100:00080000:0.0:1789761173.879098:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761173.879103:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761173.879108:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 607 00000100:00080000:0.0:1789761173.879111:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761173.879113:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789761173.879118:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761173.879127:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0e00 x1876700024579072/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789761173.879157:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761173.879190:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761173/real 1789761173] req@ffffa06c091d1500 x1876700024578816/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761198 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761173.879203:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d1500 x1876700024578816/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761198 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761173.879211:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761173.879214:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761173.879217:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024578816:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789761173.879222:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024578944:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761173.879244:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761173.879253:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761173/real 1789761173] req@ffffa06c091d2a00 x1876700024578944/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761198 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761173.879261:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d2a00 x1876700024578944/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761198 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761173.879267:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761173.879269:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761173.879271:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024578944:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789761173.879292:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024579072:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761173.879309:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761173.879321:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761173/real 1789761173] req@ffffa06c091d0e00 x1876700024579072/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761198 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761173.879335:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0e00 x1876700024579072/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761198 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761173.879343:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761173.879347:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761173.879350:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024579072:192.168.203.107@tcp:38:kworker.0 00010000:00010000:0.0:1789761174.135066:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761174.390969:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761175.159002:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761175.414968:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761176.183049:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761176.438990:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789761177.207254:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761177.462929:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761178.230967:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761178.486971:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:1.0:1789761178.998961:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761178.998970:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789761178.998974:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761178.999002:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 612 00000100:00080000:1.0:1789761178.999013:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761178.999022:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789761178.999056:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2c700 x1876700024579200/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761178.999085:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761178.999089:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761178.999091:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761178.999094:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 612 00000100:00080000:1.0:1789761178.999098:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761178.999102:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789761178.999109:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761178.999115:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024579200:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789761178.999122:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2ca80 x1876700024579328/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761178.999132:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761178.999135:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761178.999136:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761178.999139:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 612 00000100:00080000:1.0:1789761178.999142:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761178.999144:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789761178.999147:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761178.999155:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2ed80 x1876700024579456/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789761178.999170:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789761178.999208:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761178/real 1789761178] req@ffffa06c08e2c700 x1876700024579200/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761203 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761178.999221:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2c700 x1876700024579200/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761203 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761178.999229:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761178.999231:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789761178.999234:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024579200:192.168.203.107@tcp:250:kworker.0 00000100:00100000:0.0:1789761178.999251:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024579328:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789761178.999265:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789761178.999273:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761178/real 1789761178] req@ffffa06c08e2ca80 x1876700024579328/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761203 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761178.999292:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2ca80 x1876700024579328/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761203 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761178.999298:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761178.999300:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789761178.999303:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024579328:192.168.203.107@tcp:38:kworker.0 00000100:00100000:0.0:1789761178.999306:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024579456:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789761178.999320:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789761178.999331:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761178/real 1789761178] req@ffffa06c08e2ed80 x1876700024579456/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761203 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761178.999343:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2ed80 x1876700024579456/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761203 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761178.999351:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761178.999354:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789761178.999358:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024579456:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789761179.254965:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761179.510935:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761180.279005:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761180.534925:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761181.302983:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761181.558911:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761182.326963:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761182.582928:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761183.350978:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761183.606966:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789761184.118958:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761184.118973:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761184.118978:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761184.118990:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 617 00000100:00080000:0.0:1789761184.119002:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761184.119011:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789761184.119060:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d1180 x1876700024579584/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761184.119105:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761184.119108:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761184.119110:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761184.119114:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 617 00000100:00080000:0.0:1789761184.119117:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761184.119121:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789761184.119133:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761184.119152:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d3b80 x1876700024579712/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789761184.119157:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024579584:192.168.203.107@tcp:250:kworker.0 00000100:00080000:0.0:1789761184.119184:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761184.119189:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761184.119192:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761184.119198:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 617 00000100:00080000:0.0:1789761184.119204:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761184.119210:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789761184.119217:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1789761184.119237:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789761184.119244:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d2300 x1876700024579840/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789761184.119287:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761184/real 1789761184] req@ffffa06c091d1180 x1876700024579584/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761209 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761184.119301:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d1180 x1876700024579584/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761209 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761184.119310:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761184.119313:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761184.119324:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024579584:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789761184.119348:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024579712:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761184.119366:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761184.119375:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761184/real 1789761184] req@ffffa06c091d3b80 x1876700024579712/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761209 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761184.119384:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d3b80 x1876700024579712/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761209 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761184.119393:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761184.119397:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761184.119400:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024579712:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789761184.119408:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024579840:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761184.119446:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761184.119454:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761184/real 1789761184] req@ffffa06c091d2300 x1876700024579840/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761209 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761184.119462:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d2300 x1876700024579840/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761209 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761184.119467:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761184.119470:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761184.119472:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024579840:192.168.203.107@tcp:38:kworker.0 00010000:00010000:2.0:1789761184.374938:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761184.630972:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761185.398954:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761185.654965:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789761186.422964:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761186.678945:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789761187.446983:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761187.702994:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789761188.470959:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761188.726952:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789761189.238968:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761189.238976:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789761189.238979:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761189.238988:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 623 00000100:00080000:2.0:1789761189.238997:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761189.239002:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789761189.239031:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e53480 x1876700024579968/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761189.239052:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761189.239055:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761189.239056:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761189.239059:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 623 00000100:00080000:2.0:1789761189.239061:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761189.239064:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789761189.239070:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761189.239078:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50700 x1876700024580096/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761189.239086:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761189.239089:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761189.239103:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761189.239106:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 623 00000100:00080000:2.0:1789761189.239109:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1789761189.239112:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e53480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024579968:192.168.203.107@tcp:250:kworker.0 00000100:00080000:2.0:1789761189.239114:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789761189.239121:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761189.239129:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50a80 x1876700024580224/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789761189.239181:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761189.239220:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761189/real 1789761189] req@ffffa06c07e53480 x1876700024579968/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761214 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761189.239240:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e53480 x1876700024579968/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761214 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761189.239252:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761189.239267:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761189.239272:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e53480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024579968:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789761189.239291:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024580096:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761189.239320:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761189.239335:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761189/real 1789761189] req@ffffa06c07e50700 x1876700024580096/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761214 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761189.239349:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50700 x1876700024580096/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761214 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761189.239358:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761189.239362:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761189.239366:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024580096:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789761189.239400:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024580224:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761189.239424:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761189.239437:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761189/real 1789761189] req@ffffa06c07e50a80 x1876700024580224/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761214 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761189.239451:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50a80 x1876700024580224/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761214 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761189.239461:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761189.239464:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761189.239468:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024580224:192.168.203.107@tcp:38:kworker.0 00010000:00010000:0.0:1789761189.494959:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761189.750954:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761190.519122:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761190.774912:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761191.543002:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761191.798953:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761192.566999:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761192.822960:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761193.591021:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761193.847039:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:3.0:1789761194.358968:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761194.358978:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789761194.358982:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761194.358993:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 628 00000100:00080000:3.0:1789761194.359005:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761194.359012:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:3.0:1789761194.359068:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872ce00 x1876700024580352/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761194.359093:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761194.359097:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761194.359099:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761194.359103:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 628 00000100:00080000:3.0:1789761194.359108:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761194.359112:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789761194.359122:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761194.359136:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872ca80 x1876700024580480/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761194.359149:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761194.359153:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761194.359155:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761194.359160:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 628 00000100:00080000:3.0:1789761194.359164:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761194.359167:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789761194.359173:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761194.359183:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024580352:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789761194.359185:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872f100 x1876700024580608/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789761194.359284:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761194.359351:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761194/real 1789761194] req@ffffa06c1872ce00 x1876700024580352/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761219 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761194.359369:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872ce00 x1876700024580352/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761219 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761194.359379:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761194.359383:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761194.359386:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024580352:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789761194.359394:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024580480:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761194.359413:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761194.359423:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761194/real 1789761194] req@ffffa06c1872ca80 x1876700024580480/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761219 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761194.359431:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872ca80 x1876700024580480/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761219 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761194.359438:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761194.359440:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761194.359443:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024580480:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789761194.359467:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024580608:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761194.359479:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761194.359500:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761194/real 1789761194] req@ffffa06c1872f100 x1876700024580608/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761219 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761194.359509:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872f100 x1876700024580608/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761219 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761194.359515:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761194.359517:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761194.359519:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024580608:192.168.203.107@tcp:38:kworker.0 00010000:00010000:2.0:1789761194.614973:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761194.870956:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761195.639026:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761195.894914:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761196.662983:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761196.918934:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761197.687018:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761197.942968:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761198.710967:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761198.966944:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:1.0:1789761199.478961:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761199.478970:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789761199.478974:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761199.478984:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 633 00000100:00080000:1.0:1789761199.478995:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761199.479003:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789761199.479040:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2d180 x1876700024580736/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761199.479071:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761199.479075:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761199.479077:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761199.479081:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 633 00000100:00080000:1.0:1789761199.479101:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761199.479106:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789761199.479116:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761199.479125:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024580736:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789761199.479135:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2ed80 x1876700024580864/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761199.479151:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761199.479155:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761199.479156:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761199.479160:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 633 00000100:00080000:1.0:1789761199.479164:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761199.479169:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789761199.479175:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761199.479189:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2ca80 x1876700024580992/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789761199.479200:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789761199.479251:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761199/real 1789761199] req@ffffa06c08e2d180 x1876700024580736/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761224 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761199.479268:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2d180 x1876700024580736/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761224 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761199.479279:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761199.479283:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789761199.479286:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024580736:192.168.203.107@tcp:250:kworker.0 00000100:00100000:0.0:1789761199.479309:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024580864:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789761199.479329:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789761199.479340:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761199/real 1789761199] req@ffffa06c08e2ed80 x1876700024580864/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761224 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761199.479352:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2ed80 x1876700024580864/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761224 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761199.479359:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761199.479362:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789761199.479365:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024580864:192.168.203.107@tcp:38:kworker.0 00000100:00100000:0.0:1789761199.479371:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024580992:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789761199.479389:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789761199.479399:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761199/real 1789761199] req@ffffa06c08e2ca80 x1876700024580992/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761224 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761199.479424:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2ca80 x1876700024580992/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761224 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761199.479432:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761199.479435:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789761199.479438:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024580992:192.168.203.107@tcp:38:kworker.0 00010000:00010000:2.0:1789761199.734982:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761199.990962:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761200.758977:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761201.014946:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761201.782960:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761202.038911:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761202.807016:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761203.062927:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761203.830974:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761204.086948:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:1.0:1789761204.598961:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761204.598970:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789761204.598973:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761204.598981:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 638 00000100:00080000:1.0:1789761204.598989:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761204.598995:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789761204.599027:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2c700 x1876700024581120/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761204.599048:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761204.599066:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761204.599068:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761204.599072:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 638 00000100:00080000:1.0:1789761204.599076:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761204.599079:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789761204.599087:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761204.599092:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024581120:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789761204.599102:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2e300 x1876700024581248/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761204.599123:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761204.599126:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761204.599128:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761204.599130:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 638 00000100:00080000:1.0:1789761204.599153:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761204.599156:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789761204.599159:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1789761204.599164:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761204.599169:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2c000 x1876700024581376/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789761204.599218:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761204/real 1789761204] req@ffffa06c08e2c700 x1876700024581120/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761229 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761204.599234:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2c700 x1876700024581120/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761229 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761204.599242:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761204.599244:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789761204.599247:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024581120:192.168.203.107@tcp:250:kworker.0 00000100:00100000:0.0:1789761204.599265:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024581248:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789761204.599281:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789761204.599289:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761204/real 1789761204] req@ffffa06c08e2e300 x1876700024581248/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761229 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761204.599297:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2e300 x1876700024581248/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761229 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761204.599313:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761204.599315:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789761204.599317:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024581248:192.168.203.107@tcp:38:kworker.0 00000100:00100000:0.0:1789761204.599321:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024581376:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789761204.599335:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789761204.599342:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761204/real 1789761204] req@ffffa06c08e2c000 x1876700024581376/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761229 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761204.599349:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2c000 x1876700024581376/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761229 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761204.599353:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761204.599355:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789761204.599357:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024581376:192.168.203.107@tcp:38:kworker.0 00010000:00010000:2.0:1789761204.854960:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761205.110938:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761205.878978:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761206.134915:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761206.903060:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761207.158929:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761207.926987:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761208.182913:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761208.951044:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761209.206939:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:3.0:1789761209.718949:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761209.718961:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789761209.718966:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761209.718994:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 643 00000100:00080000:3.0:1789761209.719005:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761209.719012:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:3.0:1789761209.719044:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872ed80 x1876700024581504/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761209.719078:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761209.719083:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761209.719085:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761209.719089:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 643 00000100:00080000:3.0:1789761209.719093:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761209.719098:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789761209.719109:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761209.719117:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024581504:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789761209.719132:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872c700 x1876700024581632/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761209.719157:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761209.719161:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761209.719163:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761209.719167:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 643 00000100:00080000:3.0:1789761209.719171:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761209.719176:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789761209.719182:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1789761209.719187:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789761209.719198:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872dc00 x1876700024581760/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789761209.719234:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761209/real 1789761209] req@ffffa06c1872ed80 x1876700024581504/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761234 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761209.719248:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872ed80 x1876700024581504/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761234 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761209.719258:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761209.719261:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789761209.719270:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024581504:192.168.203.107@tcp:250:kworker.0 00000100:00100000:0.0:1789761209.719289:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024581632:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789761209.719305:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789761209.719313:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761209/real 1789761209] req@ffffa06c1872c700 x1876700024581632/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761234 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761209.719321:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872c700 x1876700024581632/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761234 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761209.719326:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761209.719328:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789761209.719330:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024581632:192.168.203.107@tcp:38:kworker.0 00000100:00100000:0.0:1789761209.719334:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024581760:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789761209.719343:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789761209.719349:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761209/real 1789761209] req@ffffa06c1872dc00 x1876700024581760/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761234 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761209.719356:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872dc00 x1876700024581760/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761234 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761209.719361:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761209.719362:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789761209.719364:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024581760:192.168.203.107@tcp:38:kworker.0 00010000:00010000:2.0:1789761209.974956:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761210.230907:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761210.998990:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761211.254936:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789761212.022951:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761212.278929:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761213.046944:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761213.302937:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789761214.070981:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761214.326994:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789761214.838953:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761214.838964:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761214.838969:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761214.838979:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 648 00000100:00080000:0.0:1789761214.838991:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761214.838999:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789761214.839041:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0000 x1876700024581888/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761214.839073:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761214.839076:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761214.839078:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761214.839081:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 648 00000100:00080000:0.0:1789761214.839084:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761214.839086:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789761214.839095:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761214.839108:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d2300 x1876700024582016/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761214.839118:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761214.839122:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761214.839123:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761214.839129:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 648 00000100:00080000:0.0:1789761214.839135:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1789761214.839137:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024581888:192.168.203.107@tcp:250:kworker.0 00000100:00080000:0.0:1789761214.839141:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789761214.839155:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761214.839190:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d3b80 x1876700024582144/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789761214.839227:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761214.839264:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761214/real 1789761214] req@ffffa06c091d0000 x1876700024581888/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761239 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761214.839277:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0000 x1876700024581888/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761239 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761214.839303:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761214.839306:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761214.839309:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024581888:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789761214.839316:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024582016:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761214.839332:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761214.839341:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761214/real 1789761214] req@ffffa06c091d2300 x1876700024582016/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761239 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761214.839349:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d2300 x1876700024582016/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761239 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761214.839354:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761214.839356:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761214.839358:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024582016:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789761214.839378:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024582144:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761214.839389:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761214.839395:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761214/real 1789761214] req@ffffa06c091d3b80 x1876700024582144/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761239 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761214.839402:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d3b80 x1876700024582144/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761239 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761214.839407:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761214.839409:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761214.839411:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024582144:192.168.203.107@tcp:38:kworker.0 00010000:00010000:1.0:1789761215.095117:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761215.350987:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761216.119084:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761216.374956:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789761217.142960:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761217.398952:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761218.167047:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761218.422959:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761219.191015:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761219.446957:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:3.0:1789761219.958951:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761219.958963:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789761219.958967:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761219.958978:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 653 00000100:00080000:3.0:1789761219.958989:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761219.958996:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:3.0:1789761219.959027:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872ea00 x1876700024582272/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761219.959057:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761219.959062:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761219.959064:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761219.959068:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 653 00000100:00080000:3.0:1789761219.959072:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761219.959077:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789761219.959087:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761219.959094:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024582272:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789761219.959099:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872f480 x1876700024582400/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761219.959112:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761219.959115:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761219.959117:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761219.959120:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 653 00000100:00080000:3.0:1789761219.959123:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761219.959138:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789761219.959142:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761219.959152:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872e680 x1876700024582528/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789761219.959201:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024582400:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789761219.959232:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024582528:192.168.203.107@tcp:38:kworker.0 00010000:00010000:0.0:1789761220.215028:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761220.470962:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761221.238980:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761221.494954:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761222.262970:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761222.518978:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000400:00000100:2.0:1789761223.031089:0:2328:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.107:988 00000400:00000100:2.0:1789761223.031126:0:2328:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.107@tcp at host 192.168.203.107 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1789761223.031140:0:2328:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.107@tcp 00000800:00000100:2.0:1789761223.031182:0:2328:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.107@tcp 00000800:00000100:2.0:1789761223.031192:0:2328:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.107@tcp 00000100:00100000:1.0:1789761223.031230:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761219/real 1789761223] req@ffffa06c1872f480 x1876700024582400/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761244 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761223.031256:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872f480 x1876700024582400/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761244 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761223.031269:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761223.031274:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761223.031279:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024582400:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789761223.031287:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761219/real 1789761223] req@ffffa06c1872e680 x1876700024582528/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761244 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761223.031299:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872e680 x1876700024582528/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761244 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761223.031307:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761223.031310:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761223.031314:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024582528:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789761223.031321:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761219/real 1789761223] req@ffffa06c1872ea00 x1876700024582272/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761244 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761223.031353:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872ea00 x1876700024582272/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761244 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761223.031361:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761223.031364:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761223.031373:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024582272:192.168.203.107@tcp:250:kworker.0 00010000:00010000:0.0:1789761223.286971:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761223.542935:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761224.310982:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761224.566926:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:3.0:1789761225.078920:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761225.078932:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789761225.078936:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761225.078947:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 658 00000100:00080000:3.0:1789761225.078958:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761225.078967:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:3.0:1789761225.079000:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872fb80 x1876700024582656/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761225.079031:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761225.079035:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761225.079037:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761225.079042:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 658 00000100:00080000:3.0:1789761225.079047:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761225.079051:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789761225.079060:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761225.079069:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024582656:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789761225.079075:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872ea00 x1876700024582784/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761225.079086:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761225.079099:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761225.079101:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761225.079104:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 658 00000100:00080000:3.0:1789761225.079108:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761225.079111:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789761225.079116:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1789761225.079125:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789761225.079126:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872e680 x1876700024582912/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789761225.079160:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761225/real 1789761225] req@ffffa06c1872fb80 x1876700024582656/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761250 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761225.079178:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872fb80 x1876700024582656/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761250 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761225.079189:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761225.079193:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761225.079198:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024582656:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789761225.079221:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024582784:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761225.079244:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761225.079259:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761225/real 1789761225] req@ffffa06c1872ea00 x1876700024582784/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761250 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761225.079271:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872ea00 x1876700024582784/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761250 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761225.079279:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761225.079289:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761225.079293:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024582784:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789761225.079299:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024582912:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761225.079330:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761225.079343:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761225/real 1789761225] req@ffffa06c1872e680 x1876700024582912/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761250 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761225.079355:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872e680 x1876700024582912/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761250 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761225.079364:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761225.079367:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761225.079370:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024582912:192.168.203.107@tcp:38:kworker.0 00010000:00010000:2.0:1789761225.334982:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761225.590939:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789761226.358987:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761226.614947:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761227.382936:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761227.638926:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761228.407027:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761228.662959:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789761229.431046:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761229.686912:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:3.0:1789761230.198957:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761230.198965:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789761230.198968:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761230.198976:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 664 00000100:00080000:3.0:1789761230.198986:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761230.198992:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:3.0:1789761230.199023:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872f480 x1876700024583040/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761230.199041:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761230.199044:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761230.199045:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761230.199048:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 664 00000100:00080000:3.0:1789761230.199050:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761230.199053:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789761230.199060:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761230.199080:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024583040:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789761230.199087:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872dc00 x1876700024583168/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761230.199097:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761230.199099:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761230.199101:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761230.199103:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 664 00000100:00080000:3.0:1789761230.199106:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761230.199109:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789761230.199113:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761230.199121:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872c700 x1876700024583296/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789761230.199150:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761230.199199:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761230/real 1789761230] req@ffffa06c1872f480 x1876700024583040/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761255 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761230.199218:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872f480 x1876700024583040/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761255 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761230.199229:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761230.199234:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761230.199239:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024583040:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789761230.199263:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024583168:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761230.199296:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761230.199311:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761230/real 1789761230] req@ffffa06c1872dc00 x1876700024583168/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761255 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761230.199326:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872dc00 x1876700024583168/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761255 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761230.199336:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761230.199339:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761230.199343:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024583168:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789761230.199350:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024583296:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761230.199371:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761230.199384:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761230/real 1789761230] req@ffffa06c1872c700 x1876700024583296/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761255 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761230.199396:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872c700 x1876700024583296/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761255 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761230.199402:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761230.199404:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761230.199408:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024583296:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789761230.454953:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761230.710965:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761231.478984:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761231.734971:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789761232.502966:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761232.758967:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761233.526999:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761233.782924:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761234.550960:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761234.806947:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:1.0:1789761235.318955:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761235.318963:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789761235.318966:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761235.318974:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 669 00000100:00080000:1.0:1789761235.318988:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761235.319012:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789761235.319049:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2d500 x1876700024583424/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761235.319080:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761235.319089:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761235.319091:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761235.319096:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 669 00000100:00080000:1.0:1789761235.319101:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761235.319108:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789761235.319118:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761235.319128:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024583424:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789761235.319134:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2c000 x1876700024583552/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761235.319144:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761235.319147:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761235.319148:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761235.319151:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 669 00000100:00080000:1.0:1789761235.319154:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761235.319157:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789761235.319160:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761235.319168:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2e300 x1876700024583680/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789761235.319184:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789761235.319218:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761235/real 1789761235] req@ffffa06c08e2d500 x1876700024583424/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761260 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761235.319231:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2d500 x1876700024583424/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761260 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761235.319238:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761235.319241:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789761235.319244:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024583424:192.168.203.107@tcp:250:kworker.0 00000100:00100000:0.0:1789761235.319260:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024583552:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789761235.319274:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789761235.319281:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761235/real 1789761235] req@ffffa06c08e2c000 x1876700024583552/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761260 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761235.319290:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2c000 x1876700024583552/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761260 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761235.319298:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761235.319310:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789761235.319314:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024583552:192.168.203.107@tcp:38:kworker.0 00000100:00100000:0.0:1789761235.319321:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024583680:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789761235.319344:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789761235.319357:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761235/real 1789761235] req@ffffa06c08e2e300 x1876700024583680/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761260 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761235.319369:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2e300 x1876700024583680/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761260 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761235.319379:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761235.319382:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789761235.319386:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024583680:192.168.203.107@tcp:38:kworker.0 00010000:00010000:0.0:1789761235.574938:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761235.830946:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761236.598961:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761236.854943:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761237.622986:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761237.878936:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789761238.646945:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761238.902939:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761239.670948:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761239.926920:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789761240.438942:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761240.438950:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789761240.438954:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761240.438962:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 674 00000100:00080000:2.0:1789761240.438972:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761240.438979:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789761240.439015:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51c00 x1876700024583808/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761240.439041:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761240.439045:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761240.439047:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761240.439051:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 674 00000100:00080000:2.0:1789761240.439054:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761240.439068:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789761240.439076:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761240.439081:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024583808:192.168.203.107@tcp:250:kworker.0 00000100:00100000:2.0:1789761240.439090:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e53800 x1876700024583936/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761240.439100:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761240.439103:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761240.439105:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761240.439107:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 674 00000100:00080000:2.0:1789761240.439110:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761240.439113:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789761240.439116:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761240.439123:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e52680 x1876700024584064/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789761240.439136:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789761240.439167:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761240/real 1789761240] req@ffffa06c07e51c00 x1876700024583808/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761265 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761240.439179:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51c00 x1876700024583808/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761265 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761240.439186:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761240.439189:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789761240.439192:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024583808:192.168.203.107@tcp:250:kworker.0 00000100:00100000:0.0:1789761240.439216:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e53800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024583936:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789761240.439233:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789761240.439241:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761240/real 1789761240] req@ffffa06c07e53800 x1876700024583936/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761265 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761240.439249:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e53800 x1876700024583936/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761265 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761240.439254:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761240.439257:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789761240.439259:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e53800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024583936:192.168.203.107@tcp:38:kworker.0 00000100:00100000:0.0:1789761240.439263:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e52680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024584064:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789761240.439273:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789761240.439281:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761240/real 1789761240] req@ffffa06c07e52680 x1876700024584064/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761265 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761240.439293:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e52680 x1876700024584064/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761265 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761240.439301:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761240.439304:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789761240.439308:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e52680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024584064:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789761240.694940:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761240.950924:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789761241.718997:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761241.975016:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761242.742948:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761242.998940:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761243.766994:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761244.022950:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789761244.791030:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761245.046918:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:3.0:1789761245.558943:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761245.558950:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789761245.558954:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761245.558965:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 679 00000100:00080000:3.0:1789761245.558974:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761245.558995:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:3.0:1789761245.559024:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872ed80 x1876700024584192/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761245.559049:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761245.559053:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761245.559055:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761245.559060:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 679 00000100:00080000:3.0:1789761245.559065:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761245.559070:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789761245.559080:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761245.559084:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024584192:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789761245.559098:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872f100 x1876700024584320/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761245.559114:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761245.559117:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761245.559120:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761245.559124:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 679 00000100:00080000:3.0:1789761245.559129:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761245.559133:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789761245.559137:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761245.559147:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872ca80 x1876700024584448/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789761245.559156:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789761245.559194:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761245/real 1789761245] req@ffffa06c1872ed80 x1876700024584192/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761270 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761245.559211:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872ed80 x1876700024584192/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761270 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761245.559221:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761245.559225:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789761245.559229:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024584192:192.168.203.107@tcp:250:kworker.0 00000100:00100000:0.0:1789761245.559251:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024584320:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789761245.559274:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789761245.559286:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761245/real 1789761245] req@ffffa06c1872f100 x1876700024584320/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761270 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761245.559311:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872f100 x1876700024584320/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761270 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761245.559319:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761245.559323:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789761245.559326:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024584320:192.168.203.107@tcp:38:kworker.0 00000100:00100000:0.0:1789761245.559333:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024584448:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789761245.559368:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789761245.559381:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761245/real 1789761245] req@ffffa06c1872ca80 x1876700024584448/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761270 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761245.559408:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872ca80 x1876700024584448/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761270 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761245.559416:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761245.559419:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789761245.559423:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024584448:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789761245.814949:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761246.070921:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761246.838969:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761247.094971:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789761247.863044:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761248.118972:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761248.886975:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761249.142974:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761249.910959:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761250.167009:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789761250.678977:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761250.678988:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761250.678993:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761250.679004:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 684 00000100:00080000:0.0:1789761250.679017:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761250.679024:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789761250.679072:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d1180 x1876700024584576/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761250.679107:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761250.679111:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761250.679112:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761250.679116:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 684 00000100:00080000:0.0:1789761250.679120:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761250.679123:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789761250.679133:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761250.679144:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0e00 x1876700024584704/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761250.679181:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761250.679189:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:1.0:1789761250.679191:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024584576:192.168.203.107@tcp:250:kworker.0 00000100:00080000:0.0:1789761250.679192:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761250.679206:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 684 00000100:00080000:0.0:1789761250.679216:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761250.679221:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789761250.679227:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761250.679239:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d2a00 x1876700024584832/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789761250.679281:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761250.679339:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761250/real 1789761250] req@ffffa06c091d1180 x1876700024584576/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761275 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761250.679358:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d1180 x1876700024584576/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761275 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761250.679370:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761250.679374:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761250.679379:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024584576:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789761250.679412:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024584704:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761250.679436:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761250.679450:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761250/real 1789761250] req@ffffa06c091d0e00 x1876700024584704/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761275 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761250.679463:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0e00 x1876700024584704/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761275 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761250.679473:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761250.679477:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761250.679481:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024584704:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789761250.679488:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024584832:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761250.679523:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761250.679534:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761250/real 1789761250] req@ffffa06c091d2a00 x1876700024584832/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761275 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761250.679547:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d2a00 x1876700024584832/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761275 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761250.679557:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761250.679561:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761250.679565:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024584832:192.168.203.107@tcp:38:kworker.0 00010000:00010000:0.0:1789761250.935095:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761251.190999:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761251.958962:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761252.214974:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761252.983025:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761253.238950:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789761254.007014:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761254.262992:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761255.031020:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761255.286981:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789761255.798949:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761255.798969:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761255.798973:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761255.798984:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 689 00000100:00080000:0.0:1789761255.798995:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761255.799002:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789761255.799037:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d1500 x1876700024584960/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761255.799064:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761255.799068:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761255.799070:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761255.799074:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 689 00000100:00080000:0.0:1789761255.799091:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761255.799096:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789761255.799105:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024584960:192.168.203.107@tcp:250:kworker.0 00000008:00100000:0.0:1789761255.799107:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761255.799126:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d2680 x1876700024585088/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761255.799140:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761255.799144:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761255.799146:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761255.799150:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 689 00000100:00080000:0.0:1789761255.799154:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761255.799158:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000800:00000100:1.0:1789761255.799163:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000008:00100000:0.0:1789761255.799167:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761255.799182:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0380 x1876700024585216/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789761255.799202:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761255/real 1789761255] req@ffffa06c091d1500 x1876700024584960/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761280 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761255.799216:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d1500 x1876700024584960/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761280 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761255.799224:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761255.799233:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761255.799236:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024584960:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789761255.799258:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024585088:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761255.799275:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761255.799296:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761255/real 1789761255] req@ffffa06c091d2680 x1876700024585088/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761280 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761255.799317:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d2680 x1876700024585088/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761280 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761255.799322:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761255.799325:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761255.799327:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024585088:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789761255.799331:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024585216:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761255.799345:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761255.799352:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761255/real 1789761255] req@ffffa06c091d0380 x1876700024585216/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761280 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761255.799359:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0380 x1876700024585216/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761280 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761255.799364:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761255.799366:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761255.799369:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024585216:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789761256.054977:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761256.311001:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789761257.078981:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761257.334933:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761258.102954:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761258.358938:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761259.126953:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761259.382966:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761260.150958:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761260.406952:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789761260.918979:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761260.918987:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789761260.918991:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761260.919000:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 694 00000100:00080000:2.0:1789761260.919010:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761260.919016:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789761260.919046:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51f80 x1876700024585344/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761260.919072:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761260.919075:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761260.919076:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761260.919079:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 694 00000100:00080000:2.0:1789761260.919081:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761260.919083:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789761260.919091:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761260.919098:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50380 x1876700024585472/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761260.919107:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761260.919110:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761260.919111:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761260.919114:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 694 00000100:00080000:2.0:1789761260.919116:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761260.919119:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789761260.919128:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024585344:192.168.203.107@tcp:250:kworker.0 00000008:00100000:2.0:1789761260.919138:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761260.919155:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e53b80 x1876700024585600/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789761260.919189:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761260.919221:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761260/real 1789761260] req@ffffa06c07e51f80 x1876700024585344/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761285 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761260.919234:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51f80 x1876700024585344/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761285 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761260.919261:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761260.919264:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761260.919267:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024585344:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789761260.919273:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024585472:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761260.919290:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761260.919299:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761260/real 1789761260] req@ffffa06c07e50380 x1876700024585472/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761285 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761260.919307:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50380 x1876700024585472/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761285 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761260.919312:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761260.919314:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761260.919317:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024585472:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789761260.919347:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e53b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024585600:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761260.919367:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761260.919379:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761260/real 1789761260] req@ffffa06c07e53b80 x1876700024585600/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761285 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761260.919392:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e53b80 x1876700024585600/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761285 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761260.919398:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761260.919400:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761260.919402:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e53b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024585600:192.168.203.107@tcp:38:kworker.0 00010000:00010000:2.0:1789761261.175133:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761261.431002:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761262.198992:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761262.454943:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761263.223027:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761263.478958:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761264.246962:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761264.502932:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761265.270987:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761265.526952:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789761266.038941:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761266.038949:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761266.038952:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761266.038960:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 699 00000100:00080000:0.0:1789761266.038969:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761266.038974:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789761266.039005:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d3100 x1876700024585728/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761266.039027:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761266.039042:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761266.039043:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761266.039046:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 699 00000100:00080000:0.0:1789761266.039050:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761266.039054:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789761266.039061:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761266.039071:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0a80 x1876700024585856/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761266.039080:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761266.039084:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:1.0:1789761266.039084:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024585728:192.168.203.107@tcp:250:kworker.0 00000100:00080000:0.0:1789761266.039087:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761266.039094:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 699 00000100:00080000:0.0:1789761266.039097:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761266.039099:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789761266.039102:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761266.039111:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d1f80 x1876700024585984/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789761266.039154:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761266.039192:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761266/real 1789761266] req@ffffa06c091d3100 x1876700024585728/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761291 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761266.039226:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d3100 x1876700024585728/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761291 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761266.039237:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761266.039242:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761266.039246:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024585728:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789761266.039271:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024585856:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761266.039294:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761266.039307:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761266/real 1789761266] req@ffffa06c091d0a80 x1876700024585856/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761291 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761266.039321:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0a80 x1876700024585856/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761291 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761266.039330:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761266.039333:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761266.039337:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024585856:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789761266.039344:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024585984:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761266.039376:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761266.039388:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761266/real 1789761266] req@ffffa06c091d1f80 x1876700024585984/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761291 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761266.039401:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d1f80 x1876700024585984/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761291 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761266.039410:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761266.039414:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761266.039418:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024585984:192.168.203.107@tcp:38:kworker.0 00010000:00010000:2.0:1789761266.294971:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761266.550919:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761267.319038:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761267.574962:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789761268.343053:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761268.598964:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761269.366983:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761269.623016:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761270.391019:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761270.646998:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:3.0:1789761271.158985:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761271.158996:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789761271.159000:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761271.159012:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 704 00000100:00080000:3.0:1789761271.159024:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761271.159032:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:3.0:1789761271.159076:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872ce00 x1876700024586112/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761271.159105:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761271.159109:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761271.159111:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761271.159116:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 704 00000100:00080000:3.0:1789761271.159120:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761271.159125:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789761271.159138:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761271.159149:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024586112:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789761271.159156:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872d180 x1876700024586240/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761271.159188:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761271.159192:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761271.159216:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761271.159222:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 704 00000100:00080000:3.0:1789761271.159236:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761271.159242:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000800:00000100:1.0:1789761271.159244:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000008:00100000:3.0:1789761271.159249:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761271.159264:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872e300 x1876700024586368/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789761271.159293:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761271/real 1789761271] req@ffffa06c1872ce00 x1876700024586112/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761296 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761271.159310:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872ce00 x1876700024586112/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761296 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761271.159319:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761271.159322:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761271.159325:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024586112:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789761271.159360:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024586240:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761271.159375:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761271.159384:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761271/real 1789761271] req@ffffa06c1872d180 x1876700024586240/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761296 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761271.159403:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872d180 x1876700024586240/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761296 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761271.159413:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761271.159417:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761271.159421:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024586240:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789761271.159427:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024586368:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761271.159448:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761271.159459:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761271/real 1789761271] req@ffffa06c1872e300 x1876700024586368/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761296 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761271.159471:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872e300 x1876700024586368/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761296 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761271.159480:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761271.159483:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761271.159487:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024586368:192.168.203.107@tcp:38:kworker.0 00010000:00010000:0.0:1789761271.414939:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761271.670989:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761272.439017:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761272.694984:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789761273.463077:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761273.718960:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789761274.487001:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761274.742919:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761275.510993:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761275.766963:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789761276.279035:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761276.279047:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789761276.279051:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761276.279063:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 710 00000100:00080000:2.0:1789761276.279076:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761276.279083:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789761276.279165:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50e00 x1876700024586496/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761276.279209:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761276.279213:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761276.279215:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761276.279220:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 710 00000100:00080000:2.0:1789761276.279225:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761276.279230:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789761276.279243:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761276.279273:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51500 x1876700024586624/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789761276.279279:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024586496:192.168.203.107@tcp:250:kworker.0 00000100:00080000:2.0:1789761276.279287:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761276.279291:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761276.279292:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761276.279295:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 710 00000100:00080000:2.0:1789761276.279299:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761276.279304:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789761276.279309:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761276.279318:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50000 x1876700024586752/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789761276.279358:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761276.279407:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761276/real 1789761276] req@ffffa06c07e50e00 x1876700024586496/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761301 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761276.279422:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50e00 x1876700024586496/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761301 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761276.279449:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761276.279452:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761276.279456:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024586496:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789761276.279496:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024586624:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761276.279515:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761276.279524:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761276/real 1789761276] req@ffffa06c07e51500 x1876700024586624/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761301 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761276.279532:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51500 x1876700024586624/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761301 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761276.279538:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761276.279557:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761276.279559:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024586624:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789761276.279563:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024586752:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761276.279575:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761276.279581:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761276/real 1789761276] req@ffffa06c07e50000 x1876700024586752/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761301 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761276.279589:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50000 x1876700024586752/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761301 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761276.279594:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761276.279596:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761276.279598:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024586752:192.168.203.107@tcp:38:kworker.0 00010000:00010000:0.0:1789761276.535023:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761276.790981:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789761277.559037:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761277.814964:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761278.583030:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761278.838989:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761279.606984:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761279.862954:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761280.631036:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761280.887009:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789761281.398962:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761281.398972:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761281.398999:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761281.399009:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 715 00000100:00080000:0.0:1789761281.399022:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761281.399028:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789761281.399066:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0700 x1876700024586880/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761281.399108:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761281.399111:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761281.399113:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761281.399116:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 715 00000100:00080000:0.0:1789761281.399119:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761281.399122:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789761281.399132:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761281.399143:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d1f80 x1876700024587008/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761281.399152:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761281.399155:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761281.399157:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761281.399160:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 715 00000100:00080000:0.0:1789761281.399162:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761281.399165:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789761281.399169:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761281.399176:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0a80 x1876700024587136/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789761281.399191:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024586880:192.168.203.107@tcp:250:kworker.0 00000800:00000100:1.0:1789761281.399264:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761281.399307:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761281/real 1789761281] req@ffffa06c091d0700 x1876700024586880/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761306 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761281.399322:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0700 x1876700024586880/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761306 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761281.399331:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761281.399335:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761281.399338:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024586880:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789761281.399343:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024587008:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761281.399357:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761281.399369:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761281/real 1789761281] req@ffffa06c091d1f80 x1876700024587008/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761306 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761281.399382:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d1f80 x1876700024587008/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761306 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761281.399391:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761281.399395:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761281.399399:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024587008:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789761281.399428:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024587136:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761281.399445:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761281.399463:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761281/real 1789761281] req@ffffa06c091d0a80 x1876700024587136/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761306 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761281.399472:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0a80 x1876700024587136/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761306 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761281.399478:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761281.399480:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761281.399482:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024587136:192.168.203.107@tcp:38:kworker.0 00010000:00010000:1.0:1789761281.655062:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761281.910961:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789761282.679001:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761282.934962:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761283.703027:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761283.958980:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761284.727006:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761284.983007:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789761285.751003:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761286.006953:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789761286.518947:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761286.518957:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761286.518962:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761286.518973:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 720 00000100:00080000:0.0:1789761286.518985:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761286.518993:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789761286.519062:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d3100 x1876700024587264/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761286.519088:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761286.519092:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761286.519093:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761286.519100:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 720 00000100:00080000:0.0:1789761286.519104:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761286.519107:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789761286.519119:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761286.519130:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0380 x1876700024587392/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761286.519140:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761286.519144:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761286.519145:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761286.519152:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 720 00000100:00100000:1.0:1789761286.519157:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024587264:192.168.203.107@tcp:250:kworker.0 00000100:00080000:0.0:1789761286.519172:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761286.519177:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789761286.519182:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761286.519193:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d2680 x1876700024587520/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789761286.519283:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024587392:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789761286.519319:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024587520:192.168.203.107@tcp:38:kworker.0 00010000:00010000:1.0:1789761286.775094:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761287.031001:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761287.799001:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761288.054963:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761288.823072:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761289.079055:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000400:00000100:3.0:1789761289.591121:0:2329:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.107:988 00000400:00000100:3.0:1789761289.591177:0:2329:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.107@tcp at host 192.168.203.107 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1789761289.591213:0:2329:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.107@tcp 00000800:00000100:3.0:1789761289.591268:0:2329:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.107@tcp 00000800:00000100:3.0:1789761289.591282:0:2329:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.107@tcp 00000100:00100000:1.0:1789761289.591319:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761286/real 1789761289] req@ffffa06c091d0380 x1876700024587392/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761311 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761289.591352:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0380 x1876700024587392/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761311 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761289.591369:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761289.591375:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761289.591380:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024587392:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789761289.591390:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761286/real 1789761289] req@ffffa06c091d2680 x1876700024587520/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761311 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761289.591448:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d2680 x1876700024587520/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761311 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761289.591458:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761289.591462:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761289.591466:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024587520:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789761289.591474:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761286/real 1789761289] req@ffffa06c091d3100 x1876700024587264/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761311 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761289.591488:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d3100 x1876700024587264/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761311 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761289.591508:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761289.591512:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761289.591516:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024587264:192.168.203.107@tcp:250:kworker.0 00010000:00010000:1.0:1789761289.847056:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761290.102974:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761290.871062:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761291.126981:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789761291.638978:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761291.638989:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789761291.638994:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761291.639005:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 725 00000100:00080000:2.0:1789761291.639029:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761291.639037:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789761291.639080:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e52300 x1876700024587648/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761291.639112:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761291.639117:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761291.639119:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761291.639124:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 725 00000100:00080000:2.0:1789761291.639129:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761291.639154:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789761291.639166:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761291.639207:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51880 x1876700024587776/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761291.639222:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761291.639226:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761291.639228:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761291.639232:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 725 00000100:00080000:2.0:1789761291.639237:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761291.639243:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789761291.639249:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761291.639263:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51180 x1876700024587904/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789761291.639272:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e52300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024587648:192.168.203.107@tcp:250:kworker.0 00000800:00000100:1.0:1789761291.639372:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761291.639425:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761291/real 1789761291] req@ffffa06c07e52300 x1876700024587648/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761316 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761291.639684:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e52300 x1876700024587648/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761316 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761291.639698:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761291.639703:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761291.639718:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e52300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024587648:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789761291.639727:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024587776:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761291.639758:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761291.639784:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761291/real 1789761291] req@ffffa06c07e51880 x1876700024587776/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761316 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761291.639805:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51880 x1876700024587776/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761316 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761291.639815:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761291.639820:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761291.639824:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024587776:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789761291.639891:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024587904:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761291.639915:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761291.639928:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761291/real 1789761291] req@ffffa06c07e51180 x1876700024587904/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761316 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761291.639940:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51180 x1876700024587904/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761316 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761291.639948:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761291.639951:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761291.639955:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024587904:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789761291.894971:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761292.150940:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761292.918997:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761293.175026:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761293.943013:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761294.198986:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761294.966970:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761295.222998:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761295.990995:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761296.247005:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789761296.759013:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761296.759021:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761296.759024:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761296.759032:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 730 00000100:00080000:0.0:1789761296.759040:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761296.759046:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789761296.759079:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d1500 x1876700024588032/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761296.759103:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761296.759106:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761296.759107:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761296.759110:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 730 00000100:00080000:0.0:1789761296.759112:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761296.759114:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789761296.759121:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761296.759130:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d2a00 x1876700024588160/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761296.759139:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761296.759157:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761296.759159:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1789761296.759159:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024588032:192.168.203.107@tcp:250:kworker.0 00000100:00080000:0.0:1789761296.759166:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 730 00000100:00080000:0.0:1789761296.759171:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761296.759174:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789761296.759177:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761296.759187:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0e00 x1876700024588288/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789761296.759229:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761296.759272:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761296/real 1789761296] req@ffffa06c091d1500 x1876700024588032/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761321 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761296.759291:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d1500 x1876700024588032/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761321 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761296.759302:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761296.759306:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761296.759311:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024588032:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789761296.759319:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024588160:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761296.759354:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761296.759370:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761296/real 1789761296] req@ffffa06c091d2a00 x1876700024588160/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761321 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761296.759384:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d2a00 x1876700024588160/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761321 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761296.759393:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761296.759397:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761296.759406:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024588160:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789761296.759429:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024588288:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761296.759449:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761296.759461:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761296/real 1789761296] req@ffffa06c091d0e00 x1876700024588288/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761321 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761296.759497:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0e00 x1876700024588288/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761321 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761296.759506:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761296.759509:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761296.759513:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024588288:192.168.203.107@tcp:38:kworker.0 00010000:00010000:2.0:1789761297.015030:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761297.270953:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761298.039033:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761298.295060:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761299.063013:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761299.319020:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761300.087063:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761300.342974:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789761301.111009:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761301.366954:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:3.0:1789761301.878964:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761301.878972:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789761301.878976:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761301.878984:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 735 00000100:00080000:3.0:1789761301.878993:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761301.878999:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:3.0:1789761301.879030:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872d880 x1876700024588416/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761301.879050:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761301.879052:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761301.879067:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761301.879070:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 735 00000100:00080000:3.0:1789761301.879074:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761301.879077:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789761301.879085:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761301.879095:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872f800 x1876700024588544/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789761301.879102:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024588416:192.168.203.107@tcp:250:kworker.0 00000100:00080000:3.0:1789761301.879107:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761301.879110:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761301.879111:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761301.879115:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 735 00000100:00080000:3.0:1789761301.879117:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761301.879119:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789761301.879122:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761301.879131:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872df80 x1876700024588672/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789761301.879157:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761301.879190:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761301/real 1789761301] req@ffffa06c1872d880 x1876700024588416/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761326 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761301.879208:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872d880 x1876700024588416/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761326 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761301.879219:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761301.879223:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761301.879228:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024588416:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789761301.879251:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024588544:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761301.879274:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761301.879286:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761301/real 1789761301] req@ffffa06c1872f800 x1876700024588544/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761326 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761301.879315:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872f800 x1876700024588544/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761326 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761301.879325:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761301.879329:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761301.879333:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024588544:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789761301.879340:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024588672:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761301.879360:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761301.879372:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761301/real 1789761301] req@ffffa06c1872df80 x1876700024588672/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761326 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761301.879384:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872df80 x1876700024588672/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761326 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761301.879393:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761301.879396:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761301.879400:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024588672:192.168.203.107@tcp:38:kworker.0 00010000:00010000:1.0:1789761302.135034:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761302.390954:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761303.159083:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761303.414951:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761304.183010:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761304.438950:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761305.207019:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761305.462960:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789761306.231006:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761306.486963:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789761306.998971:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761306.999007:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789761306.999012:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761306.999024:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 740 00000100:00080000:2.0:1789761306.999035:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761306.999042:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789761306.999082:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51180 x1876700024588800/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761306.999113:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761306.999117:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761306.999118:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761306.999122:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 740 00000100:00080000:2.0:1789761306.999124:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761306.999127:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789761306.999136:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761306.999147:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51880 x1876700024588928/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761306.999157:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761306.999160:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761306.999163:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1789761306.999165:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024588800:192.168.203.107@tcp:250:kworker.0 00000100:00080000:2.0:1789761306.999168:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 740 00000100:00080000:2.0:1789761306.999173:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761306.999176:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789761306.999181:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761306.999189:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e52300 x1876700024589056/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789761306.999248:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761306.999297:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761306/real 1789761306] req@ffffa06c07e51180 x1876700024588800/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761331 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761306.999312:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51180 x1876700024588800/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761331 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761306.999321:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761306.999324:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761306.999327:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024588800:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789761306.999350:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024588928:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761306.999366:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761306.999375:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761306/real 1789761306] req@ffffa06c07e51880 x1876700024588928/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761331 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761306.999384:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51880 x1876700024588928/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761331 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761306.999390:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761306.999392:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761306.999394:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024588928:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789761306.999398:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e52300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024589056:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761306.999409:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761306.999414:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761306/real 1789761306] req@ffffa06c07e52300 x1876700024589056/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761331 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761306.999433:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e52300 x1876700024589056/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761331 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761306.999439:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761306.999441:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761306.999443:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e52300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024589056:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789761307.254972:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761307.510934:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761308.278983:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761308.534978:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761309.302969:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761309.558975:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761310.326955:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761310.582996:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761311.351009:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761311.606953:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789761312.119001:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761312.119012:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761312.119016:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761312.119028:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 745 00000100:00080000:0.0:1789761312.119040:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761312.119048:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789761312.119094:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d1180 x1876700024589184/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761312.119120:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761312.119123:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761312.119124:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761312.119128:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 745 00000100:00080000:0.0:1789761312.119132:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761312.119136:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789761312.119169:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761312.119188:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d3b80 x1876700024589312/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789761312.119196:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024589184:192.168.203.107@tcp:250:kworker.0 00000100:00080000:0.0:1789761312.119205:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761312.119209:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761312.119210:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761312.119214:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 745 00000100:00080000:0.0:1789761312.119217:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761312.119222:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789761312.119226:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761312.119236:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d2300 x1876700024589440/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789761312.119281:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761312.119328:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761312/real 1789761312] req@ffffa06c091d1180 x1876700024589184/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761337 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761312.119349:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d1180 x1876700024589184/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761337 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761312.119361:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761312.119365:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761312.119370:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024589184:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789761312.119396:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024589312:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761312.119423:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761312.119452:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761312/real 1789761312] req@ffffa06c091d3b80 x1876700024589312/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761337 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761312.119466:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d3b80 x1876700024589312/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761337 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761312.119476:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761312.119480:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761312.119484:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024589312:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789761312.119491:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024589440:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761312.119514:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761312.119527:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761312/real 1789761312] req@ffffa06c091d2300 x1876700024589440/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761337 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761312.119540:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d2300 x1876700024589440/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761337 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761312.119549:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761312.119553:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761312.119556:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024589440:192.168.203.107@tcp:38:kworker.0 00010000:00010000:2.0:1789761312.375063:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761312.630945:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761313.398970:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761313.654983:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761314.422978:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761314.678950:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761315.447040:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761315.702952:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789761316.470979:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761316.726930:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789761317.239006:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761317.239021:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789761317.239026:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761317.239041:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 751 00000100:00080000:2.0:1789761317.239054:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761317.239063:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789761317.239114:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50000 x1876700024589568/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761317.239149:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761317.239153:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761317.239156:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761317.239160:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 751 00000100:00080000:2.0:1789761317.239163:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761317.239168:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789761317.239180:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761317.239193:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51500 x1876700024589696/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761317.239207:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761317.239213:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:1.0:1789761317.239216:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024589568:192.168.203.107@tcp:250:kworker.0 00000100:00080000:2.0:1789761317.239216:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761317.239226:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 751 00000100:00080000:2.0:1789761317.239234:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761317.239259:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789761317.239265:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761317.239280:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50e00 x1876700024589824/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789761317.239305:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761317.239367:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761317/real 1789761317] req@ffffa06c07e50000 x1876700024589568/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761342 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761317.239382:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50000 x1876700024589568/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761342 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761317.239390:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761317.239394:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761317.239396:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024589568:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789761317.239420:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024589696:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761317.239446:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761317.239460:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761317/real 1789761317] req@ffffa06c07e51500 x1876700024589696/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761342 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761317.239469:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51500 x1876700024589696/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761342 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761317.239474:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761317.239477:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761317.239479:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024589696:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789761317.239483:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024589824:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761317.239498:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761317.239505:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761317/real 1789761317] req@ffffa06c07e50e00 x1876700024589824/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761342 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761317.239512:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50e00 x1876700024589824/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761342 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761317.239529:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761317.239532:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761317.239534:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024589824:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789761317.494967:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761317.750950:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761318.518986:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761318.774938:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789761319.543002:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761319.798999:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761320.567014:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761320.822960:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761321.590969:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761321.846983:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789761322.358930:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761322.358939:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761322.358942:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761322.358951:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 756 00000100:00080000:0.0:1789761322.358960:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761322.358966:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789761322.358996:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0000 x1876700024589952/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761322.359020:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761322.359023:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761322.359037:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761322.359040:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 756 00000100:00080000:0.0:1789761322.359043:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761322.359046:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789761322.359054:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761322.359065:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d2300 x1876700024590080/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761322.359075:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761322.359078:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:1.0:1789761322.359078:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024589952:192.168.203.107@tcp:250:kworker.0 00000100:00080000:0.0:1789761322.359082:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761322.359089:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 756 00000100:00080000:0.0:1789761322.359091:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761322.359094:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789761322.359097:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761322.359106:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d3b80 x1876700024590208/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789761322.359137:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761322.359172:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761322/real 1789761322] req@ffffa06c091d0000 x1876700024589952/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761347 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761322.359185:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0000 x1876700024589952/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761347 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761322.359192:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761322.359195:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761322.359198:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024589952:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789761322.359214:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024590080:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761322.359227:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761322.359234:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761322/real 1789761322] req@ffffa06c091d2300 x1876700024590080/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761347 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761322.359242:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d2300 x1876700024590080/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761347 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761322.359254:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761322.359256:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761322.359259:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024590080:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789761322.359263:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024590208:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761322.359275:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761322.359282:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761322/real 1789761322] req@ffffa06c091d3b80 x1876700024590208/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761347 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761322.359289:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d3b80 x1876700024590208/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761347 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761322.359294:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761322.359296:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761322.359298:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024590208:192.168.203.107@tcp:38:kworker.0 00010000:00010000:1.0:1789761322.614943:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761322.870928:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761323.638972:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761323.894943:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761324.662956:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761324.918928:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761325.686947:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761325.942934:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761326.710953:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761326.966943:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789761327.478964:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761327.478973:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789761327.478976:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761327.478985:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 761 00000100:00080000:2.0:1789761327.478993:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761327.478998:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789761327.479029:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e53b80 x1876700024590336/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761327.479053:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761327.479056:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761327.479057:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761327.479060:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 761 00000100:00080000:2.0:1789761327.479062:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761327.479065:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789761327.479071:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761327.479079:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50380 x1876700024590464/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761327.479087:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761327.479089:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761327.479104:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761327.479107:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 761 00000100:00080000:2.0:1789761327.479110:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761327.479113:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789761327.479118:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761327.479126:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51f80 x1876700024590592/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789761327.479155:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e53b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024590336:192.168.203.107@tcp:250:kworker.0 00000800:00000100:1.0:1789761327.479236:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761327.479281:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761327/real 1789761327] req@ffffa06c07e53b80 x1876700024590336/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761352 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761327.479316:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e53b80 x1876700024590336/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761352 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761327.479326:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761327.479330:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761327.479333:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e53b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024590336:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789761327.479340:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024590464:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761327.479358:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761327.479384:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761327/real 1789761327] req@ffffa06c07e50380 x1876700024590464/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761352 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761327.479394:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50380 x1876700024590464/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761352 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761327.479400:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761327.479402:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761327.479405:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024590464:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789761327.479409:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024590592:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761327.479423:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761327.479430:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761327/real 1789761327] req@ffffa06c07e51f80 x1876700024590592/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761352 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761327.479439:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51f80 x1876700024590592/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761352 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761327.479455:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761327.479459:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761327.479461:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024590592:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789761327.735000:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761327.990973:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761328.759001:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761329.014905:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761329.782964:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761330.038957:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761330.806959:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761331.062940:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789761331.830951:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761332.086939:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789761332.598944:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761332.598953:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761332.598956:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761332.598964:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 766 00000100:00080000:0.0:1789761332.598973:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761332.598979:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789761332.599011:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d1180 x1876700024590720/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761332.599043:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761332.599046:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761332.599048:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761332.599051:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 766 00000100:00080000:0.0:1789761332.599054:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761332.599057:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789761332.599064:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761332.599074:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0e00 x1876700024590848/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761332.599083:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761332.599086:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761332.599087:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761332.599092:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 766 00000100:00100000:1.0:1789761332.599094:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024590720:192.168.203.107@tcp:250:kworker.0 00000100:00080000:0.0:1789761332.599095:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761332.599101:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789761332.599107:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761332.599116:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d2a00 x1876700024590976/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789761332.599159:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761332.599194:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761332/real 1789761332] req@ffffa06c091d1180 x1876700024590720/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761357 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761332.599208:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d1180 x1876700024590720/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761357 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761332.599215:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761332.599218:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761332.599221:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024590720:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789761332.599226:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024590848:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761332.599240:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761332.599248:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761332/real 1789761332] req@ffffa06c091d0e00 x1876700024590848/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761357 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761332.599267:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0e00 x1876700024590848/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761357 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761332.599273:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761332.599275:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761332.599277:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024590848:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789761332.599296:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024590976:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761332.599308:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761332.599315:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761332/real 1789761332] req@ffffa06c091d2a00 x1876700024590976/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761357 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761332.599322:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d2a00 x1876700024590976/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761357 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761332.599327:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761332.599329:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761332.599331:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024590976:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789761332.854966:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761333.110923:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761333.878964:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761334.134952:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789761334.902967:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761335.158949:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761335.926972:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761336.182984:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761336.950959:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761337.206973:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789761337.718911:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761337.718919:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761337.718923:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761337.718930:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 771 00000100:00080000:0.0:1789761337.718954:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761337.718961:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789761337.718988:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d1500 x1876700024591104/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761337.719012:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761337.719015:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761337.719016:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761337.719019:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 771 00000100:00080000:0.0:1789761337.719021:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761337.719024:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789761337.719031:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761337.719039:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d3100 x1876700024591232/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761337.719047:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761337.719050:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761337.719052:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761337.719054:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 771 00000100:00080000:0.0:1789761337.719057:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761337.719059:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789761337.719063:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761337.719066:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024591104:192.168.203.107@tcp:250:kworker.0 00000100:00100000:0.0:1789761337.719072:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d2680 x1876700024591360/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789761337.719130:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761337.719161:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761337/real 1789761337] req@ffffa06c091d1500 x1876700024591104/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761362 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761337.719175:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d1500 x1876700024591104/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761362 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761337.719183:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761337.719186:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761337.719189:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024591104:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789761337.719194:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024591232:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761337.719207:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761337.719215:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761337/real 1789761337] req@ffffa06c091d3100 x1876700024591232/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761362 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761337.719228:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d3100 x1876700024591232/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761362 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761337.719236:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761337.719240:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761337.719243:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024591232:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789761337.719269:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024591360:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761337.719291:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761337.719300:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761337/real 1789761337] req@ffffa06c091d2680 x1876700024591360/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761362 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761337.719326:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d2680 x1876700024591360/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761362 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761337.719335:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761337.719338:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761337.719342:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024591360:192.168.203.107@tcp:38:kworker.0 00010000:00010000:1.0:1789761337.974980:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761338.230917:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761338.999017:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761339.254993:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761340.022958:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761340.278921:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761341.046963:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761341.302950:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761342.070949:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761342.326951:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789761342.838944:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761342.838967:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761342.838970:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761342.838978:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 776 00000100:00080000:0.0:1789761342.838987:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761342.838993:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789761342.839029:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0380 x1876700024591488/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761342.839057:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761342.839060:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761342.839062:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761342.839065:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 776 00000100:00080000:0.0:1789761342.839068:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761342.839071:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789761342.839078:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761342.839089:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0a80 x1876700024591616/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761342.839097:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761342.839101:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761342.839103:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761342.839107:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 776 00000100:00080000:0.0:1789761342.839111:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1789761342.839117:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024591488:192.168.203.107@tcp:250:kworker.0 00000100:00080000:0.0:1789761342.839119:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789761342.839128:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761342.839144:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d1f80 x1876700024591744/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789761342.839177:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761342.839209:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761342/real 1789761342] req@ffffa06c091d0380 x1876700024591488/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761367 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761342.839222:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0380 x1876700024591488/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761367 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761342.839230:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761342.839233:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761342.839236:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024591488:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789761342.839241:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024591616:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761342.839264:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761342.839272:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761342/real 1789761342] req@ffffa06c091d0a80 x1876700024591616/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761367 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761342.839280:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0a80 x1876700024591616/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761367 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761342.839289:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761342.839293:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761342.839297:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024591616:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789761342.839321:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024591744:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761342.839344:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761342.839357:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761342/real 1789761342] req@ffffa06c091d1f80 x1876700024591744/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761367 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761342.839371:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d1f80 x1876700024591744/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761367 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761342.839381:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761342.839384:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761342.839388:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024591744:192.168.203.107@tcp:38:kworker.0 00010000:00010000:1.0:1789761343.095033:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761343.350951:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789761344.118995:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761344.374943:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761345.142971:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761345.398942:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761346.166985:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761346.422917:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789761347.191081:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761347.446986:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:1.0:1789761347.959019:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761347.959027:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789761347.959030:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761347.959037:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 781 00000100:00080000:1.0:1789761347.959046:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761347.959051:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789761347.959085:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2c700 x1876700024591872/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761347.959108:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761347.959111:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761347.959112:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761347.959115:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 781 00000100:00080000:1.0:1789761347.959117:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761347.959119:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789761347.959127:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761347.959135:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2ca80 x1876700024592000/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761347.959144:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761347.959148:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761347.959151:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1789761347.959157:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024591872:192.168.203.107@tcp:250:kworker.0 00000100:00080000:1.0:1789761347.959158:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 781 00000100:00080000:1.0:1789761347.959165:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761347.959172:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789761347.959193:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761347.959208:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2ed80 x1876700024592128/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789761347.959228:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789761347.959271:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761347/real 1789761347] req@ffffa06c08e2c700 x1876700024591872/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761372 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761347.959290:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2c700 x1876700024591872/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761372 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761347.959301:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761347.959305:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789761347.959309:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024591872:192.168.203.107@tcp:250:kworker.0 00000100:00100000:0.0:1789761347.959317:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024592000:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789761347.959341:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789761347.959355:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761347/real 1789761347] req@ffffa06c08e2ca80 x1876700024592000/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761372 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761347.959369:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2ca80 x1876700024592000/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761372 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761347.959378:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761347.959382:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789761347.959385:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024592000:192.168.203.107@tcp:38:kworker.0 00000100:00100000:0.0:1789761347.959412:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024592128:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789761347.959452:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789761347.959465:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761347/real 1789761347] req@ffffa06c08e2ed80 x1876700024592128/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761372 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761347.959478:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2ed80 x1876700024592128/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761372 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761347.959487:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761347.959490:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789761347.959494:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024592128:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789761348.215006:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761348.470947:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761349.239115:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761349.494981:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761350.262968:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761350.518931:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761351.286953:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761351.542958:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761352.310991:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761352.566901:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789761353.078932:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761353.078941:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761353.078944:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761353.078952:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 786 00000100:00080000:0.0:1789761353.078962:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761353.078967:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789761353.078996:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0700 x1876700024592256/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761353.079016:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761353.079019:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761353.079020:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761353.079023:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 786 00000100:00080000:0.0:1789761353.079026:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761353.079028:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789761353.079036:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761353.079049:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024592256:192.168.203.107@tcp:250:kworker.0 00000100:00100000:0.0:1789761353.079066:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d1f80 x1876700024592384/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761353.079095:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761353.079099:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761353.079101:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761353.079106:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 786 00000100:00080000:0.0:1789761353.079112:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761353.079117:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789761353.079124:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761353.079135:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024592384:192.168.203.107@tcp:38:kworker.0 00000100:00100000:0.0:1789761353.079142:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0a80 x1876700024592512/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789761353.079180:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024592512:192.168.203.107@tcp:38:kworker.0 00010000:00010000:2.0:1789761353.334982:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761353.590930:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761354.359015:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761354.614904:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761355.382992:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761355.638948:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000400:00000100:2.0:1789761356.151077:0:2330:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.107:988 00000400:00000100:2.0:1789761356.151110:0:2330:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.107@tcp at host 192.168.203.107 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1789761356.151123:0:2330:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.107@tcp 00000800:00000100:2.0:1789761356.151155:0:2330:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.107@tcp 00000800:00000100:2.0:1789761356.151165:0:2330:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.107@tcp 00000100:00100000:1.0:1789761356.151205:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761353/real 1789761356] req@ffffa06c091d0a80 x1876700024592512/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761378 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761356.151230:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0a80 x1876700024592512/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761378 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761356.151241:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761356.151245:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761356.151249:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024592512:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789761356.151257:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761353/real 1789761356] req@ffffa06c091d1f80 x1876700024592384/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761378 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761356.151284:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d1f80 x1876700024592384/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761378 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761356.151291:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761356.151295:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761356.151298:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024592384:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789761356.151305:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761353/real 1789761356] req@ffffa06c091d0700 x1876700024592256/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761378 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761356.151317:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0700 x1876700024592256/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761378 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761356.151325:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761356.151328:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761356.151331:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024592256:192.168.203.107@tcp:250:kworker.0 00010000:00010000:2.0:1789761356.407053:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761356.662907:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761357.431030:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761357.686923:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789761358.198989:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761358.198997:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789761358.199000:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761358.199008:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 792 00000100:00080000:2.0:1789761358.199016:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761358.199021:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789761358.199055:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e52680 x1876700024592640/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761358.199087:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761358.199102:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761358.199103:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761358.199107:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 792 00000100:00080000:2.0:1789761358.199112:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761358.199118:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789761358.199128:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761358.199143:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e53800 x1876700024592768/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761358.199160:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761358.199165:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761358.199167:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1789761358.199171:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e52680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024592640:192.168.203.107@tcp:250:kworker.0 00000100:00080000:2.0:1789761358.199175:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 792 00000100:00080000:2.0:1789761358.199180:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761358.199193:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789761358.199201:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761358.199218:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51c00 x1876700024592896/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789761358.199289:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761358.199361:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761358/real 1789761358] req@ffffa06c07e52680 x1876700024592640/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761383 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761358.199386:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e52680 x1876700024592640/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761383 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761358.199401:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761358.199436:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761358.199441:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e52680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024592640:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789761358.199478:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e53800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024592768:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761358.199509:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761358.199540:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761358/real 1789761358] req@ffffa06c07e53800 x1876700024592768/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761383 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761358.199555:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e53800 x1876700024592768/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761383 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761358.199565:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761358.199568:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761358.199572:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e53800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024592768:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789761358.199579:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024592896:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761358.199601:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761358.199620:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761358/real 1789761358] req@ffffa06c07e51c00 x1876700024592896/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761383 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761358.199638:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51c00 x1876700024592896/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761383 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761358.199648:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761358.199652:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761358.199656:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024592896:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789761358.454996:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761358.711007:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761359.478976:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761359.734942:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761360.503000:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761360.758983:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761361.526980:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761361.782958:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761362.550974:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761362.806951:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789761363.318919:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761363.318926:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789761363.318930:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761363.318938:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 797 00000100:00080000:2.0:1789761363.318947:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761363.318952:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789761363.319013:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50a80 x1876700024593024/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761363.319038:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761363.319042:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761363.319044:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761363.319049:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 797 00000100:00080000:2.0:1789761363.319053:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761363.319059:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789761363.319069:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761363.319080:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024593024:192.168.203.107@tcp:250:kworker.0 00000100:00100000:2.0:1789761363.319086:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50700 x1876700024593152/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761363.319097:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761363.319100:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761363.319101:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761363.319104:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 797 00000100:00080000:2.0:1789761363.319106:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761363.319110:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789761363.319113:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761363.319121:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e53480 x1876700024593280/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789761363.319136:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761363.319170:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761363/real 1789761363] req@ffffa06c07e50a80 x1876700024593024/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761388 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761363.319182:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50a80 x1876700024593024/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761388 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761363.319190:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761363.319193:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761363.319195:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024593024:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789761363.319212:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024593152:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761363.319226:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761363.319233:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761363/real 1789761363] req@ffffa06c07e50700 x1876700024593152/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761388 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761363.319241:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50700 x1876700024593152/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761388 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761363.319246:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761363.319248:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761363.319250:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024593152:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789761363.319261:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e53480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024593280:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761363.319274:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761363.319281:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761363/real 1789761363] req@ffffa06c07e53480 x1876700024593280/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761388 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761363.319288:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e53480 x1876700024593280/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761388 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761363.319294:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761363.319295:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761363.319297:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e53480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024593280:192.168.203.107@tcp:38:kworker.0 00010000:00010000:2.0:1789761363.574962:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761363.830946:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761364.599015:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761364.854949:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761365.623027:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761365.878911:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789761366.647046:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761366.902974:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761367.670972:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761367.926939:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789761368.438987:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761368.438995:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761368.438998:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761368.439006:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 802 00000100:00080000:0.0:1789761368.439015:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761368.439020:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789761368.439052:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0380 x1876700024593408/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761368.439072:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761368.439075:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761368.439077:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761368.439079:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 802 00000100:00080000:0.0:1789761368.439095:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761368.439099:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789761368.439106:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761368.439119:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d2680 x1876700024593536/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789761368.439119:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024593408:192.168.203.107@tcp:250:kworker.0 00000100:00080000:0.0:1789761368.439137:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761368.439140:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761368.439141:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761368.439144:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 802 00000100:00080000:0.0:1789761368.439146:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761368.439148:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789761368.439151:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761368.439159:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d3100 x1876700024593664/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789761368.439186:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761368.439218:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761368/real 1789761368] req@ffffa06c091d0380 x1876700024593408/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761393 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761368.439231:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0380 x1876700024593408/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761393 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761368.439239:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761368.439242:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761368.439245:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024593408:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789761368.439274:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024593536:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761368.439299:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761368.439314:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761368/real 1789761368] req@ffffa06c091d2680 x1876700024593536/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761393 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761368.439327:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d2680 x1876700024593536/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761393 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761368.439336:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761368.439340:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761368.439344:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024593536:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789761368.439351:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024593664:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761368.439371:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761368.439380:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761368/real 1789761368] req@ffffa06c091d3100 x1876700024593664/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761393 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761368.439388:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d3100 x1876700024593664/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761393 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761368.439393:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761368.439396:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761368.439398:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024593664:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789761368.694981:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761368.950933:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761369.718995:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761369.974993:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761370.742967:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761370.998945:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761371.766993:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761372.022924:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761372.790976:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761373.046924:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789761373.558962:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761373.558971:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789761373.558974:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761373.558982:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 807 00000100:00080000:2.0:1789761373.558990:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761373.558995:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789761373.559025:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e52a00 x1876700024593792/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761373.559045:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761373.559048:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761373.559050:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761373.559052:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 807 00000100:00080000:2.0:1789761373.559055:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761373.559061:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789761373.559068:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761373.559077:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e53100 x1876700024593920/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761373.559087:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1789761373.559091:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e52a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024593792:192.168.203.107@tcp:250:kworker.0 00000100:00080000:2.0:1789761373.559091:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761373.559095:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761373.559102:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 807 00000100:00080000:2.0:1789761373.559104:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761373.559106:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789761373.559109:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761373.559131:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e52d80 x1876700024594048/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789761373.559150:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761373.559182:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761373/real 1789761373] req@ffffa06c07e52a00 x1876700024593792/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761398 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761373.559194:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e52a00 x1876700024593792/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761398 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761373.559202:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761373.559205:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761373.559208:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e52a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024593792:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789761373.559225:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e53100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024593920:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761373.559237:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761373.559245:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761373/real 1789761373] req@ffffa06c07e53100 x1876700024593920/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761398 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761373.559252:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e53100 x1876700024593920/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761398 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761373.559258:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761373.559260:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761373.559262:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e53100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024593920:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789761373.559265:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e52d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024594048:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761373.559275:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761373.559289:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761373/real 1789761373] req@ffffa06c07e52d80 x1876700024594048/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761398 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761373.559297:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e52d80 x1876700024594048/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761398 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761373.559302:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761373.559304:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761373.559306:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e52d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024594048:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789761373.815024:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761374.070913:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761374.839000:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761375.094974:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761375.862976:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761376.118978:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761376.886956:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761377.142936:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761377.911048:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761378.166983:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789761378.678967:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761378.678977:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789761378.678982:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761378.678993:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 812 00000100:00080000:2.0:1789761378.679004:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761378.679012:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789761378.679056:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e52d80 x1876700024594176/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761378.679082:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761378.679085:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761378.679086:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761378.679095:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 812 00000100:00080000:2.0:1789761378.679098:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761378.679102:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789761378.679112:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761378.679123:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e53100 x1876700024594304/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761378.679133:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1789761378.679134:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e52d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024594176:192.168.203.107@tcp:250:kworker.0 00000100:00080000:2.0:1789761378.679138:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761378.679142:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761378.679146:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 812 00000100:00080000:2.0:1789761378.679167:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761378.679171:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789761378.679176:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761378.679186:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e52a00 x1876700024594432/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789761378.679219:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761378.679279:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761378/real 1789761378] req@ffffa06c07e52d80 x1876700024594176/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761403 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761378.679296:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e52d80 x1876700024594176/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761403 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761378.679305:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761378.679308:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761378.679312:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e52d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024594176:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789761378.679341:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e53100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024594304:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761378.679381:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761378.679406:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761378/real 1789761378] req@ffffa06c07e53100 x1876700024594304/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761403 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761378.679421:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e53100 x1876700024594304/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761403 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761378.679430:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761378.679432:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761378.679435:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e53100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024594304:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789761378.679440:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e52a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024594432:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761378.679458:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761378.679466:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761378/real 1789761378] req@ffffa06c07e52a00 x1876700024594432/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761403 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761378.679475:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e52a00 x1876700024594432/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761403 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761378.679481:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761378.679483:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761378.679486:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e52a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024594432:192.168.203.107@tcp:38:kworker.0 00010000:00010000:1.0:1789761378.935000:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761379.190961:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761379.958960:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761380.214970:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761380.982992:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761381.238965:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761382.006990:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761382.262937:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761383.030944:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761383.286934:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789761383.798970:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761383.798981:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789761383.798985:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761383.798995:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 817 00000100:00080000:2.0:1789761383.799007:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761383.799015:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789761383.799064:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e53480 x1876700024594560/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761383.799089:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761383.799092:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761383.799094:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761383.799097:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 817 00000100:00080000:2.0:1789761383.799099:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761383.799102:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789761383.799111:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761383.799119:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50700 x1876700024594688/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761383.799127:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761383.799151:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761383.799152:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761383.799155:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 817 00000100:00080000:2.0:1789761383.799158:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761383.799161:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789761383.799165:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761383.799175:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50a80 x1876700024594816/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789761383.799179:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e53480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024594560:192.168.203.107@tcp:250:kworker.0 00000800:00000100:1.0:1789761383.799271:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761383.799315:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761383/real 1789761383] req@ffffa06c07e53480 x1876700024594560/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761408 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761383.799345:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e53480 x1876700024594560/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761408 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761383.799354:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761383.799357:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761383.799361:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e53480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024594560:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789761383.799367:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024594688:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761383.799385:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761383.799394:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761383/real 1789761383] req@ffffa06c07e50700 x1876700024594688/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761408 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761383.799402:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50700 x1876700024594688/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761408 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761383.799408:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761383.799410:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761383.799412:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024594688:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789761383.799435:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024594816:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761383.799452:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761383.799463:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761383/real 1789761383] req@ffffa06c07e50a80 x1876700024594816/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761408 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761383.799490:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50a80 x1876700024594816/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761408 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761383.799500:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761383.799503:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761383.799507:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024594816:192.168.203.107@tcp:38:kworker.0 00010000:00010000:2.0:1789761384.055015:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761384.311001:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761385.078978:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761385.334909:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761386.103007:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761386.358950:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761387.127013:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761387.382905:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761388.150989:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761388.406944:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789761388.918929:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761388.918940:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789761388.918944:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761388.918955:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 822 00000100:00080000:2.0:1789761388.918967:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761388.918975:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789761388.919012:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51c00 x1876700024594944/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761388.919042:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761388.919046:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761388.919048:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761388.919053:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 822 00000100:00080000:2.0:1789761388.919057:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761388.919062:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789761388.919072:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761388.919085:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024594944:192.168.203.107@tcp:250:kworker.0 00000100:00100000:2.0:1789761388.919108:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e53800 x1876700024595072/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761388.919124:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761388.919128:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761388.919131:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761388.919135:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 822 00000100:00080000:2.0:1789761388.919141:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761388.919146:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000800:00000100:1.0:1789761388.919150:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000008:00100000:2.0:1789761388.919155:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761388.919172:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e52680 x1876700024595200/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789761388.919190:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761388/real 1789761388] req@ffffa06c07e51c00 x1876700024594944/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761413 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761388.919207:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51c00 x1876700024594944/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761413 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761388.919215:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761388.919226:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761388.919229:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024594944:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789761388.919250:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e53800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024595072:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761388.919274:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761388.919302:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761388/real 1789761388] req@ffffa06c07e53800 x1876700024595072/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761413 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761388.919321:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e53800 x1876700024595072/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761413 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761388.919331:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761388.919335:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761388.919339:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e53800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024595072:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789761388.919345:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e52680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024595200:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761388.919362:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761388.919370:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761388/real 1789761388] req@ffffa06c07e52680 x1876700024595200/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761413 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761388.919378:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e52680 x1876700024595200/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761413 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761388.919383:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761388.919386:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761388.919388:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e52680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024595200:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789761389.174973:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761389.430915:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789761390.199056:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761390.454973:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761391.223056:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761391.478919:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761392.246989:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761392.503070:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761393.271015:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761393.526927:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:1.0:1789761394.038959:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761394.039027:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789761394.039031:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761394.039041:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 827 00000100:00080000:1.0:1789761394.039052:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761394.039059:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789761394.039104:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2d180 x1876700024595328/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761394.039127:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761394.039130:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761394.039131:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761394.039134:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 827 00000100:00080000:1.0:1789761394.039137:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761394.039140:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789761394.039147:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761394.039156:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2ed80 x1876700024595456/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761394.039165:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761394.039167:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761394.039169:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761394.039171:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 827 00000100:00080000:1.0:1789761394.039173:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761394.039175:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789761394.039179:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761394.039185:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024595328:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789761394.039188:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2ca80 x1876700024595584/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789761394.039263:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789761394.039305:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761394/real 1789761394] req@ffffa06c08e2d180 x1876700024595328/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761419 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761394.039325:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2d180 x1876700024595328/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761419 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761394.039337:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761394.039341:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789761394.039354:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024595328:192.168.203.107@tcp:250:kworker.0 00000100:00100000:0.0:1789761394.039362:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024595456:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789761394.039395:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789761394.039410:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761394/real 1789761394] req@ffffa06c08e2ed80 x1876700024595456/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761419 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761394.039423:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2ed80 x1876700024595456/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761419 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761394.039432:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761394.039441:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789761394.039445:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024595456:192.168.203.107@tcp:38:kworker.0 00000100:00100000:0.0:1789761394.039489:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024595584:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789761394.039513:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789761394.039525:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761394/real 1789761394] req@ffffa06c08e2ca80 x1876700024595584/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761419 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761394.039538:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2ca80 x1876700024595584/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761419 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761394.039547:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761394.039551:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789761394.039555:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024595584:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789761394.294955:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761394.550947:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761395.318970:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761395.574906:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761396.343010:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761396.598918:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761397.366977:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761397.622954:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761398.390947:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761398.646928:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:1.0:1789761399.158955:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761399.158966:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789761399.158970:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761399.158980:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 832 00000100:00080000:1.0:1789761399.158992:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761399.158999:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789761399.159037:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2c700 x1876700024595712/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761399.159065:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761399.159069:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761399.159071:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761399.159076:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 832 00000100:00080000:1.0:1789761399.159080:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761399.159085:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789761399.159095:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761399.159110:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024595712:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789761399.159112:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2e300 x1876700024595840/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761399.159126:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761399.159129:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761399.159131:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761399.159133:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 832 00000100:00080000:1.0:1789761399.159136:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761399.159152:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789761399.159157:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1789761399.159168:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761399.159169:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2c000 x1876700024595968/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789761399.159230:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761399/real 1789761399] req@ffffa06c08e2c700 x1876700024595712/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761424 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761399.159251:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2c700 x1876700024595712/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761424 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761399.159264:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761399.159268:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789761399.159272:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024595712:192.168.203.107@tcp:250:kworker.0 00000100:00100000:0.0:1789761399.159300:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024595840:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789761399.159322:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789761399.159332:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761399/real 1789761399] req@ffffa06c08e2e300 x1876700024595840/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761424 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761399.159340:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2e300 x1876700024595840/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761424 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761399.159346:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761399.159349:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789761399.159351:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024595840:192.168.203.107@tcp:38:kworker.0 00000100:00100000:0.0:1789761399.159355:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024595968:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789761399.159368:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789761399.159374:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761399/real 1789761399] req@ffffa06c08e2c000 x1876700024595968/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761424 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761399.159381:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2c000 x1876700024595968/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761424 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761399.159397:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761399.159399:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789761399.159401:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024595968:192.168.203.107@tcp:38:kworker.0 00010000:00010000:0.0:1789761399.414984:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761399.670941:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789761400.438943:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761400.694924:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761401.462952:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761401.718934:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789761402.487049:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761402.742956:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761403.511024:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761403.766953:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789761404.279022:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761404.279030:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789761404.279050:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761404.279059:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 838 00000100:00080000:2.0:1789761404.279069:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761404.279075:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789761404.279111:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51f80 x1876700024596096/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761404.279144:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761404.279148:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761404.279150:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761404.279156:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 838 00000100:00080000:2.0:1789761404.279161:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761404.279178:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789761404.279188:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761404.279202:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024596096:192.168.203.107@tcp:250:kworker.0 00000100:00100000:2.0:1789761404.279203:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50380 x1876700024596224/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761404.279219:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761404.279222:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761404.279224:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761404.279226:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 838 00000100:00080000:2.0:1789761404.279229:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761404.279233:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789761404.279239:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761404.279250:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e53b80 x1876700024596352/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789761404.279262:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789761404.279300:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761404/real 1789761404] req@ffffa06c07e51f80 x1876700024596096/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761429 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761404.279320:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51f80 x1876700024596096/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761429 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761404.279331:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761404.279335:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789761404.279338:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024596096:192.168.203.107@tcp:250:kworker.0 00000100:00100000:0.0:1789761404.279357:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024596224:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789761404.279375:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789761404.279383:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761404/real 1789761404] req@ffffa06c07e50380 x1876700024596224/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761429 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761404.279392:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50380 x1876700024596224/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761429 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761404.279407:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761404.279410:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789761404.279412:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024596224:192.168.203.107@tcp:38:kworker.0 00000100:00100000:0.0:1789761404.279419:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e53b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024596352:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789761404.279439:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789761404.279451:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761404/real 1789761404] req@ffffa06c07e53b80 x1876700024596352/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761429 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761404.279463:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e53b80 x1876700024596352/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761429 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761404.279478:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761404.279481:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789761404.279484:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e53b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024596352:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789761404.535035:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761404.790915:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789761405.558964:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761405.814944:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761406.583017:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761406.839000:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761407.607017:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761407.862971:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761408.631035:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761408.886989:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789761409.398981:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761409.399001:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761409.399005:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761409.399013:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 843 00000100:00080000:0.0:1789761409.399021:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761409.399027:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789761409.399056:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d1500 x1876700024596480/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761409.399079:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761409.399098:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761409.399100:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761409.399105:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 843 00000100:00080000:0.0:1789761409.399110:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761409.399116:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789761409.399126:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761409.399145:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024596480:192.168.203.107@tcp:250:kworker.0 00000100:00100000:0.0:1789761409.399159:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d2a00 x1876700024596608/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761409.399175:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761409.399178:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761409.399180:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761409.399184:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 843 00000100:00080000:0.0:1789761409.399208:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761409.399211:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789761409.399216:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1789761409.399221:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789761409.399229:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0e00 x1876700024596736/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789761409.399268:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761409/real 1789761409] req@ffffa06c091d1500 x1876700024596480/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761434 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761409.399288:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d1500 x1876700024596480/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761434 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761409.399300:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761409.399304:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761409.399308:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024596480:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789761409.399330:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024596608:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761409.399352:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761409.399365:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761409/real 1789761409] req@ffffa06c091d2a00 x1876700024596608/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761434 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761409.399377:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d2a00 x1876700024596608/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761434 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761409.399393:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761409.399397:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761409.399401:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024596608:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789761409.399407:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024596736:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761409.399428:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761409.399440:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761409/real 1789761409] req@ffffa06c091d0e00 x1876700024596736/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761434 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761409.399452:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0e00 x1876700024596736/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761434 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761409.399461:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761409.399464:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761409.399468:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024596736:192.168.203.107@tcp:38:kworker.0 00010000:00010000:1.0:1789761409.655017:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761409.910928:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761410.678937:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761410.934928:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789761411.702952:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761411.958923:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761412.727056:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761412.982953:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761413.751024:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761414.006943:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789761414.518928:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761414.518936:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789761414.518940:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761414.518947:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 848 00000100:00080000:2.0:1789761414.518956:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761414.518979:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789761414.519015:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50e00 x1876700024596864/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761414.519044:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761414.519049:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761414.519051:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761414.519056:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 848 00000100:00080000:2.0:1789761414.519061:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761414.519067:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789761414.519077:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761414.519087:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024596864:192.168.203.107@tcp:250:kworker.0 00000100:00100000:2.0:1789761414.519106:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51500 x1876700024596992/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761414.519118:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761414.519121:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761414.519122:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761414.519138:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 848 00000100:00080000:2.0:1789761414.519142:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761414.519146:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789761414.519151:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761414.519160:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50000 x1876700024597120/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789761414.519166:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789761414.519210:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761414/real 1789761414] req@ffffa06c07e50e00 x1876700024596864/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761439 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761414.519230:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50e00 x1876700024596864/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761439 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761414.519250:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761414.519255:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789761414.519260:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024596864:192.168.203.107@tcp:250:kworker.0 00000100:00100000:0.0:1789761414.519283:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024596992:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789761414.519309:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789761414.519322:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761414/real 1789761414] req@ffffa06c07e51500 x1876700024596992/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761439 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761414.519334:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51500 x1876700024596992/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761439 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761414.519344:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761414.519348:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789761414.519351:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024596992:192.168.203.107@tcp:38:kworker.0 00000100:00100000:0.0:1789761414.519358:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024597120:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789761414.519378:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789761414.519390:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761414/real 1789761414] req@ffffa06c07e50000 x1876700024597120/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761439 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761414.519402:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50000 x1876700024597120/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761439 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761414.519411:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761414.519414:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789761414.519417:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024597120:192.168.203.107@tcp:38:kworker.0 00010000:00010000:0.0:1789761414.774974:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761415.030933:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761415.798945:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761416.054933:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761416.823044:0:253:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:0.0:1789761416.823104:0:5427:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1789761416.823107:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761416.823117:0:5427:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c110cbc00/0x59bc99bb6194815b lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x1daf3002fe9e0a20 expref: -99 pid: 9931 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1789761416.823197:0:5427:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c110cbc00/0x59bc99bb6194815b lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x1daf3002fe9e0a20 expref: -99 pid: 9931 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1789761416.823203:0:5427:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1789761416.823231:0:5427:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffffa06c18a1e000 lock: ffffa06c110cbc00/0x59bc99bb6194815b lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x1daf3002fe9e0a20 expref: -99 pid: 9931 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1789761416.823257:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761416.823259:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761416.823261:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789761416.823281:0:2341:0:(client.c:1845:ptlrpc_send_new_req()) @@@ req waiting for recovery: (FULL != DISCONN) req@ffffa06c091d1180 x1876700024597248/t0(0) o103->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:17/18 lens 328/224 e 0 to 0 dl 0 ref 1 fl Rpc:WQU/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761417.078919:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761417.846942:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761418.102949:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761418.870989:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761419.126937:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:3.0:1789761419.638914:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761419.638923:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789761419.638926:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761419.638934:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 853 00000100:00080000:3.0:1789761419.638943:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761419.638949:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:3.0:1789761419.638979:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872c380 x1876700024597376/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761419.639000:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761419.639003:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761419.639004:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761419.639007:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 853 00000100:00080000:3.0:1789761419.639009:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761419.639011:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789761419.639018:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761419.639045:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024597376:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789761419.639101:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872df80 x1876700024597504/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761419.639116:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761419.639119:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761419.639121:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761419.639126:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 853 00000100:00080000:3.0:1789761419.639131:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761419.639136:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789761419.639142:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761419.639156:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872f800 x1876700024597632/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789761419.639159:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024597504:192.168.203.107@tcp:38:kworker.0 00000100:00100000:0.0:1789761419.639207:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024597632:192.168.203.107@tcp:38:kworker.0 00010000:00010000:1.0:1789761419.894951:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761420.150932:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761420.918980:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761421.174975:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761421.942934:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761422.198915:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000400:00000100:1.0:1789761422.711066:0:2331:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.107:988 00000400:00000100:1.0:1789761422.711106:0:2331:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.107@tcp at host 192.168.203.107 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1789761422.711123:0:2331:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.107@tcp 00000800:00000100:1.0:1789761422.711157:0:2331:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.107@tcp 00000800:00000100:1.0:1789761422.711167:0:2331:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.107@tcp 00000100:00100000:0.0:1789761422.711205:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761419/real 1789761422] req@ffffa06c1872f800 x1876700024597632/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761444 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761422.711232:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872f800 x1876700024597632/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761444 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761422.711250:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761422.711255:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789761422.711260:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024597632:192.168.203.107@tcp:38:kworker.0 00000100:00100000:0.0:1789761422.711283:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761419/real 1789761422] req@ffffa06c1872df80 x1876700024597504/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761444 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761422.711298:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872df80 x1876700024597504/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761444 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761422.711307:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761422.711311:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789761422.711315:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024597504:192.168.203.107@tcp:38:kworker.0 00000100:00100000:0.0:1789761422.711322:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761419/real 1789761422] req@ffffa06c1872c380 x1876700024597376/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761444 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761422.711336:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872c380 x1876700024597376/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761444 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761422.711346:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761422.711349:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789761422.711353:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024597376:192.168.203.107@tcp:250:kworker.0 00010000:00010000:1.0:1789761422.966949:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761423.222952:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789761423.991045:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761424.246908:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789761424.758925:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761424.758936:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761424.758940:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761424.758958:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 858 00000100:00080000:0.0:1789761424.758970:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761424.758978:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789761424.759027:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d3b80 x1876700024597760/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761424.759062:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761424.759066:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761424.759069:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761424.759075:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 858 00000100:00080000:0.0:1789761424.759080:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761424.759085:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789761424.759094:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024597760:192.168.203.107@tcp:250:kworker.0 00000008:00100000:0.0:1789761424.759096:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761424.759116:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d2300 x1876700024597888/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761424.759129:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761424.759134:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761424.759136:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761424.759140:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 858 00000100:00080000:0.0:1789761424.759153:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761424.759158:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000800:00000100:1.0:1789761424.759164:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000008:00100000:0.0:1789761424.759166:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761424.759181:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0000 x1876700024598016/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789761424.759236:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761424/real 1789761424] req@ffffa06c091d3b80 x1876700024597760/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761449 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761424.759261:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d3b80 x1876700024597760/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761449 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761424.759272:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761424.759277:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761424.759281:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024597760:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789761424.759305:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024597888:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761424.759331:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761424.759345:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761424/real 1789761424] req@ffffa06c091d2300 x1876700024597888/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761449 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761424.759358:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d2300 x1876700024597888/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761449 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761424.759367:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761424.759379:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761424.759383:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024597888:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789761424.759390:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024598016:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761424.759417:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761424.759430:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761424/real 1789761424] req@ffffa06c091d0000 x1876700024598016/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761449 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761424.759442:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0000 x1876700024598016/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761449 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761424.759451:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761424.759454:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761424.759457:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024598016:192.168.203.107@tcp:38:kworker.0 00010000:00010000:2.0:1789761425.014963:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761425.270951:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761426.039050:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761426.294927:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761427.062975:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761427.318958:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761428.087001:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761428.342919:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761429.111008:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761429.366924:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789761429.878973:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761429.878992:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789761429.878996:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761429.879004:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 863 00000100:00080000:2.0:1789761429.879014:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761429.879021:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789761429.879052:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e52300 x1876700024598144/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761429.879090:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761429.879093:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761429.879094:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761429.879098:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 863 00000100:00080000:2.0:1789761429.879101:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761429.879105:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789761429.879113:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761429.879117:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e52300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024598144:192.168.203.107@tcp:250:kworker.0 00000100:00100000:2.0:1789761429.879126:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51880 x1876700024598272/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761429.879137:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761429.879139:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761429.879140:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761429.879143:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 863 00000100:00080000:2.0:1789761429.879146:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761429.879149:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789761429.879153:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761429.879160:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51180 x1876700024598400/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789761429.879174:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761429.879208:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761429/real 1789761429] req@ffffa06c07e52300 x1876700024598144/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761454 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761429.879221:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e52300 x1876700024598144/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761454 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761429.879240:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761429.879243:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761429.879246:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e52300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024598144:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789761429.879268:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024598272:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761429.879293:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761429.879307:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761429/real 1789761429] req@ffffa06c07e51880 x1876700024598272/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761454 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761429.879319:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51880 x1876700024598272/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761454 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761429.879325:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761429.879327:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761429.879329:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024598272:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789761429.879334:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024598400:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761429.879349:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761429.879355:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761429/real 1789761429] req@ffffa06c07e51180 x1876700024598400/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761454 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761429.879363:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51180 x1876700024598400/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761454 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761429.879368:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761429.879370:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761429.879372:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024598400:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789761430.134969:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761430.390910:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761431.158944:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761431.414935:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761432.183035:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761432.438933:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761433.207037:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761433.462966:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761434.231052:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761434.486982:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789761434.998959:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761434.998967:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789761434.998986:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761434.998994:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 868 00000100:00080000:2.0:1789761434.999003:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761434.999008:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789761434.999038:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51180 x1876700024598528/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761434.999063:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761434.999066:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761434.999067:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761434.999070:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 868 00000100:00080000:2.0:1789761434.999072:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761434.999075:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789761434.999082:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761434.999094:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51880 x1876700024598656/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761434.999107:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1789761434.999110:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024598528:192.168.203.107@tcp:250:kworker.0 00000100:00080000:2.0:1789761434.999113:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761434.999118:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761434.999123:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 868 00000100:00080000:2.0:1789761434.999128:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761434.999132:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789761434.999139:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761434.999153:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e52300 x1876700024598784/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789761434.999165:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761434.999199:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761434/real 1789761434] req@ffffa06c07e51180 x1876700024598528/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761459 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761434.999212:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51180 x1876700024598528/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761459 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761434.999220:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761434.999223:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761434.999226:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024598528:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789761434.999243:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024598656:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761434.999256:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761434.999264:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761434/real 1789761434] req@ffffa06c07e51880 x1876700024598656/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761459 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761434.999272:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51880 x1876700024598656/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761459 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761434.999277:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761434.999280:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761434.999282:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024598656:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789761434.999292:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e52300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024598784:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761434.999305:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761434.999312:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761434/real 1789761434] req@ffffa06c07e52300 x1876700024598784/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761459 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761434.999319:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e52300 x1876700024598784/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761459 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761434.999325:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761434.999327:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761434.999329:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e52300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024598784:192.168.203.107@tcp:38:kworker.0 00010000:00010000:2.0:1789761435.255072:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761435.510926:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761436.278975:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761436.534914:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761437.303010:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761437.558936:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761438.326983:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761438.583107:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761439.350998:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761439.606952:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789761440.119037:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761440.119051:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761440.119057:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761440.119069:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 873 00000100:00080000:0.0:1789761440.119103:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761440.119112:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789761440.119168:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0000 x1876700024598912/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761440.119201:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761440.119205:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761440.119208:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761440.119213:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 873 00000100:00080000:0.0:1789761440.119217:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761440.119222:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789761440.119234:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761440.119253:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d2300 x1876700024599040/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789761440.119253:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024598912:192.168.203.107@tcp:250:kworker.0 00000100:00080000:0.0:1789761440.119273:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761440.119278:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761440.119280:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761440.119285:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 873 00000100:00080000:0.0:1789761440.119297:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761440.119303:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789761440.119309:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761440.119322:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d3b80 x1876700024599168/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789761440.119342:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761440.119388:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761440/real 1789761440] req@ffffa06c091d0000 x1876700024598912/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761465 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761440.119405:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0000 x1876700024598912/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761465 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761440.119414:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761440.119417:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761440.119421:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024598912:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789761440.119457:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024599040:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761440.119477:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761440.119488:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761440/real 1789761440] req@ffffa06c091d2300 x1876700024599040/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761465 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761440.119502:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d2300 x1876700024599040/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761465 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761440.119511:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761440.119514:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761440.119518:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024599040:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789761440.119524:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024599168:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761440.119545:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761440.119556:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761440/real 1789761440] req@ffffa06c091d3b80 x1876700024599168/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761465 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761440.119569:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d3b80 x1876700024599168/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761465 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761440.119577:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761440.119581:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761440.119584:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024599168:192.168.203.107@tcp:38:kworker.0 00010000:00010000:2.0:1789761440.375016:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761440.630973:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789761441.399005:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761441.654952:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761442.422966:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761442.678958:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761443.446980:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761443.702923:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761444.471020:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761444.726954:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789761445.238950:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761445.238959:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761445.238962:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761445.238971:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 879 00000100:00080000:0.0:1789761445.238984:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761445.238990:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789761445.239025:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0e00 x1876700024599296/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761445.239045:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761445.239049:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761445.239051:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761445.239056:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 879 00000100:00080000:0.0:1789761445.239058:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761445.239061:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789761445.239069:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761445.239077:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d2a00 x1876700024599424/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761445.239086:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761445.239088:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761445.239089:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761445.239091:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 879 00000100:00080000:0.0:1789761445.239093:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761445.239096:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789761445.239099:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761445.239105:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024599296:192.168.203.107@tcp:250:kworker.0 00000100:00100000:0.0:1789761445.239122:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d1500 x1876700024599552/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789761445.239167:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761445.239199:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761445/real 1789761445] req@ffffa06c091d0e00 x1876700024599296/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761470 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761445.239213:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0e00 x1876700024599296/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761470 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761445.239221:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761445.239223:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761445.239226:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024599296:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789761445.239231:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024599424:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761445.239245:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761445.239252:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761445/real 1789761445] req@ffffa06c091d2a00 x1876700024599424/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761470 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761445.239260:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d2a00 x1876700024599424/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761470 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761445.239265:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761445.239267:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761445.239269:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024599424:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789761445.239287:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024599552:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761445.239297:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761445.239312:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761445/real 1789761445] req@ffffa06c091d1500 x1876700024599552/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761470 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761445.239331:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d1500 x1876700024599552/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761470 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761445.239337:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761445.239339:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761445.239341:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024599552:192.168.203.107@tcp:38:kworker.0 00010000:00010000:1.0:1789761445.494960:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761445.750925:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789761446.519004:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761446.774947:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761447.542974:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761447.798924:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761448.567021:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761448.822963:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789761449.590978:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761449.846925:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:3.0:1789761450.358926:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761450.358935:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789761450.358938:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761450.358946:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 884 00000100:00080000:3.0:1789761450.358954:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761450.358960:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:3.0:1789761450.358995:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872d880 x1876700024599680/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761450.359025:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761450.359028:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761450.359029:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761450.359032:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 884 00000100:00080000:3.0:1789761450.359035:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761450.359037:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789761450.359045:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761450.359054:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872e300 x1876700024599808/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761450.359063:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761450.359066:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761450.359067:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761450.359072:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 884 00000100:00100000:1.0:1789761450.359074:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024599680:192.168.203.107@tcp:250:kworker.0 00000100:00080000:3.0:1789761450.359075:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761450.359080:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789761450.359089:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761450.359101:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872d180 x1876700024599936/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789761450.359144:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761450.359189:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761450/real 1789761450] req@ffffa06c1872d880 x1876700024599680/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761475 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761450.359209:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872d880 x1876700024599680/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761475 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761450.359220:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761450.359224:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761450.359229:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024599680:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789761450.359253:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024599808:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761450.359279:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761450.359289:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761450/real 1789761450] req@ffffa06c1872e300 x1876700024599808/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761475 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761450.359299:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872e300 x1876700024599808/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761475 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761450.359308:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761450.359311:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761450.359315:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024599808:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789761450.359321:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024599936:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761450.359343:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761450.359355:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761450/real 1789761450] req@ffffa06c1872d180 x1876700024599936/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761475 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761450.359365:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872d180 x1876700024599936/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761475 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761450.359371:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761450.359373:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761450.359375:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024599936:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789761450.615029:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761450.870945:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761451.638991:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761451.894912:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761452.662939:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761452.918960:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761453.686945:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761453.942897:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761454.711050:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761454.966982:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:3.0:1789761455.478951:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761455.478960:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789761455.478964:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761455.478990:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 889 00000100:00080000:3.0:1789761455.479001:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761455.479007:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:3.0:1789761455.479047:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872ce00 x1876700024600064/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761455.479077:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761455.479081:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761455.479083:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761455.479088:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 889 00000100:00080000:3.0:1789761455.479093:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761455.479097:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789761455.479107:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761455.479128:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872ca80 x1876700024600192/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761455.479143:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761455.479147:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761455.479154:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1789761455.479158:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024600064:192.168.203.107@tcp:250:kworker.0 00000100:00080000:3.0:1789761455.479162:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 889 00000100:00080000:3.0:1789761455.479171:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761455.479177:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789761455.479183:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761455.479198:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872f100 x1876700024600320/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789761455.479247:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761455.479307:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761455/real 1789761455] req@ffffa06c1872ce00 x1876700024600064/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761480 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761455.479330:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872ce00 x1876700024600064/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761480 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761455.479344:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761455.479348:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761455.479351:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024600064:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789761455.479379:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024600192:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761455.479400:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761455.479410:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761455/real 1789761455] req@ffffa06c1872ca80 x1876700024600192/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761480 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761455.479419:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872ca80 x1876700024600192/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761480 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761455.479425:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761455.479428:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761455.479430:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024600192:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789761455.479435:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024600320:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761455.479447:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761455.479453:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761455/real 1789761455] req@ffffa06c1872f100 x1876700024600320/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761480 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761455.479480:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872f100 x1876700024600320/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761480 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761455.479489:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761455.479493:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761455.479496:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024600320:192.168.203.107@tcp:38:kworker.0 00010000:00010000:2.0:1789761455.735063:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761455.990954:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789761456.759042:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761457.014943:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761457.783025:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761458.038963:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761458.806958:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761459.062928:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789761459.831052:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761460.086946:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:3.0:1789761460.598941:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761460.598950:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789761460.598953:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761460.598961:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 894 00000100:00080000:3.0:1789761460.598970:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761460.598976:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:3.0:1789761460.599023:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872ed80 x1876700024600448/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761460.599049:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761460.599052:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761460.599054:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761460.599057:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 894 00000100:00080000:3.0:1789761460.599059:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761460.599063:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789761460.599071:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761460.599082:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872c700 x1876700024600576/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761460.599091:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761460.599095:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761460.599097:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1789761460.599098:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024600448:192.168.203.107@tcp:250:kworker.0 00000100:00080000:3.0:1789761460.599102:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 894 00000100:00080000:3.0:1789761460.599108:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761460.599111:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789761460.599129:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761460.599140:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872dc00 x1876700024600704/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789761460.599169:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761460.599226:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761460/real 1789761460] req@ffffa06c1872ed80 x1876700024600448/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761485 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761460.599247:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872ed80 x1876700024600448/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761485 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761460.599256:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761460.599259:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761460.599263:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024600448:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789761460.599283:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024600576:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761460.599301:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761460.599332:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761460/real 1789761460] req@ffffa06c1872c700 x1876700024600576/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761485 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761460.599347:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872c700 x1876700024600576/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761485 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761460.599356:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761460.599360:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761460.599364:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024600576:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789761460.599370:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024600704:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761460.599389:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761460.599398:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761460/real 1789761460] req@ffffa06c1872dc00 x1876700024600704/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761485 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761460.599405:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872dc00 x1876700024600704/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761485 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761460.599410:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761460.599413:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761460.599415:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024600704:192.168.203.107@tcp:38:kworker.0 00010000:00010000:1.0:1789761460.854987:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761461.110917:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761461.878967:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761462.134958:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761462.902946:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761463.158937:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761463.926976:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761464.182946:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761464.950986:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761465.206912:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:1.0:1789761465.718974:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761465.718998:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789761465.719001:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761465.719009:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 899 00000100:00080000:1.0:1789761465.719017:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761465.719023:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789761465.719050:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2d500 x1876700024600832/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761465.719085:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761465.719089:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761465.719091:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761465.719096:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 899 00000100:00080000:1.0:1789761465.719101:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761465.719106:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789761465.719116:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761465.719133:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2c000 x1876700024600960/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789761465.719147:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024600832:192.168.203.107@tcp:250:kworker.0 00000100:00080000:1.0:1789761465.719163:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761465.719169:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761465.719170:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761465.719177:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 899 00000100:00080000:1.0:1789761465.719180:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761465.719185:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789761465.719190:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761465.719202:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2e300 x1876700024601088/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789761465.719240:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789761465.719295:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761465/real 1789761465] req@ffffa06c08e2d500 x1876700024600832/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761490 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761465.719317:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2d500 x1876700024600832/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761490 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761465.719330:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761465.719336:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789761465.719341:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024600832:192.168.203.107@tcp:250:kworker.0 00000100:00100000:0.0:1789761465.719372:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024600960:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789761465.719399:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789761465.719414:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761465/real 1789761465] req@ffffa06c08e2c000 x1876700024600960/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761490 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761465.719428:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2c000 x1876700024600960/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761490 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761465.719438:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761465.719442:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789761465.719457:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024600960:192.168.203.107@tcp:38:kworker.0 00000100:00100000:0.0:1789761465.719463:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024601088:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789761465.719481:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789761465.719489:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761465/real 1789761465] req@ffffa06c08e2e300 x1876700024601088/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761490 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761465.719500:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2e300 x1876700024601088/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761490 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761465.719508:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761465.719512:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789761465.719515:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024601088:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789761465.974932:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761466.230923:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761466.999002:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761467.254909:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761468.022956:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761468.278955:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761469.046972:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761469.302929:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761470.071033:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761470.326942:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:1.0:1789761470.838932:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761470.838955:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789761470.838959:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761470.838967:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 904 00000100:00080000:1.0:1789761470.838976:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761470.838982:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789761470.839012:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2c700 x1876700024601216/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761470.839036:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761470.839039:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761470.839041:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761470.839044:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 904 00000100:00080000:1.0:1789761470.839047:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761470.839049:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789761470.839055:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761470.839064:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2ca80 x1876700024601344/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761470.839073:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1789761470.839085:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024601216:192.168.203.107@tcp:250:kworker.0 00000100:00080000:1.0:1789761470.839093:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761470.839096:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761470.839100:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 904 00000100:00080000:1.0:1789761470.839103:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761470.839106:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789761470.839111:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761470.839122:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2ed80 x1876700024601472/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789761470.839145:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789761470.839187:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761470/real 1789761470] req@ffffa06c08e2c700 x1876700024601216/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761495 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761470.839207:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2c700 x1876700024601216/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761495 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761470.839218:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761470.839223:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789761470.839227:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024601216:192.168.203.107@tcp:250:kworker.0 00000100:00100000:0.0:1789761470.839234:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024601344:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789761470.839254:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789761470.839273:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761470/real 1789761470] req@ffffa06c08e2ca80 x1876700024601344/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761495 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761470.839282:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2ca80 x1876700024601344/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761495 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761470.839288:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761470.839290:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789761470.839292:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024601344:192.168.203.107@tcp:38:kworker.0 00000100:00100000:0.0:1789761470.839310:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024601472:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789761470.839323:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789761470.839329:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761470/real 1789761470] req@ffffa06c08e2ed80 x1876700024601472/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761495 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761470.839337:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2ed80 x1876700024601472/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761495 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761470.839342:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761470.839344:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789761470.839354:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024601472:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789761471.094988:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761471.350972:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761472.119002:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761472.374942:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789761473.143041:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761473.398983:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761474.166934:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761474.422943:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761475.190957:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761475.446952:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789761475.958949:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761475.958960:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761475.958964:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761475.958974:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 909 00000100:00080000:0.0:1789761475.958985:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761475.958993:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789761475.959042:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d3100 x1876700024601600/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761475.959065:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761475.959068:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761475.959070:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761475.959073:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 909 00000100:00080000:0.0:1789761475.959077:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761475.959080:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789761475.959087:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761475.959096:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024601600:192.168.203.107@tcp:250:kworker.0 00000100:00100000:0.0:1789761475.959099:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d2680 x1876700024601728/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761475.959112:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761475.959114:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761475.959116:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761475.959118:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 909 00000100:00080000:0.0:1789761475.959121:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761475.959123:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789761475.959126:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761475.959146:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0380 x1876700024601856/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789761475.959167:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761475.959206:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761475/real 1789761475] req@ffffa06c091d3100 x1876700024601600/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761500 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761475.959220:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d3100 x1876700024601600/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761500 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761475.959229:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761475.959232:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761475.959235:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024601600:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789761475.959266:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024601728:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761475.959291:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761475.959300:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761475/real 1789761475] req@ffffa06c091d2680 x1876700024601728/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761500 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761475.959308:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d2680 x1876700024601728/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761500 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761475.959314:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761475.959316:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761475.959318:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024601728:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789761475.959322:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024601856:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761475.959334:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761475.959339:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761475/real 1789761475] req@ffffa06c091d0380 x1876700024601856/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761500 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761475.959347:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0380 x1876700024601856/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761500 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761475.959363:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761475.959365:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761475.959369:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024601856:192.168.203.107@tcp:38:kworker.0 00010000:00010000:0.0:1789761476.215002:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761476.470936:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761477.239040:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761477.494991:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761478.263050:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761478.518948:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789761479.286950:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761479.542935:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761480.311015:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761480.566914:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:1.0:1789761481.078971:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761481.078981:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789761481.078986:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761481.078997:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 914 00000100:00080000:1.0:1789761481.079008:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761481.079016:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789761481.079060:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2d180 x1876700024601984/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761481.079093:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761481.079097:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761481.079099:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761481.079104:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 914 00000100:00080000:1.0:1789761481.079108:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761481.079113:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789761481.079123:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761481.079138:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2ed80 x1876700024602112/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789761481.079141:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024601984:192.168.203.107@tcp:250:kworker.0 00000100:00080000:1.0:1789761481.079158:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761481.079165:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761481.079169:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761481.079173:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 914 00000100:00080000:1.0:1789761481.079177:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761481.079182:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789761481.079195:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761481.079208:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2ca80 x1876700024602240/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789761481.079214:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789761481.079249:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761481/real 1789761481] req@ffffa06c08e2d180 x1876700024601984/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761506 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761481.079275:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2d180 x1876700024601984/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761506 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761481.079282:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761481.079285:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789761481.079288:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024601984:192.168.203.107@tcp:250:kworker.0 00000100:00100000:0.0:1789761481.079306:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024602112:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789761481.079320:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789761481.079328:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761481/real 1789761481] req@ffffa06c08e2ed80 x1876700024602112/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761506 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761481.079347:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2ed80 x1876700024602112/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761506 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761481.079353:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761481.079355:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789761481.079357:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024602112:192.168.203.107@tcp:38:kworker.0 00000100:00100000:0.0:1789761481.079361:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024602240:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789761481.079370:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789761481.079376:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761481/real 1789761481] req@ffffa06c08e2ca80 x1876700024602240/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761506 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761481.079393:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2ca80 x1876700024602240/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761506 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761481.079399:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761481.079401:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789761481.079403:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024602240:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789761481.334947:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761481.590917:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761482.358950:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761482.614912:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761483.382963:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761483.638953:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761484.407012:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761484.662908:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761485.431054:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761485.686933:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789761486.199016:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761486.199030:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761486.199035:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761486.199048:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 920 00000100:00080000:0.0:1789761486.199062:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761486.199072:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789761486.199123:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0700 x1876700024602368/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761486.199162:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761486.199167:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761486.199169:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761486.199175:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 920 00000100:00080000:0.0:1789761486.199180:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761486.199187:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789761486.199199:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761486.199217:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024602368:192.168.203.107@tcp:250:kworker.0 00000100:00100000:0.0:1789761486.199219:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d1f80 x1876700024602496/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761486.199271:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761486.199274:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761486.199276:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761486.199282:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 920 00000100:00080000:0.0:1789761486.199286:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761486.199290:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789761486.199296:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761486.199314:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0a80 x1876700024602624/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789761486.199334:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024602496:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789761486.199373:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024602624:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789761486.454945:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761486.710916:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761487.479024:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761487.734929:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761488.502951:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761488.758916:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000400:00000100:3.0:1789761489.271091:0:2328:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.107:988 00000400:00000100:3.0:1789761489.271131:0:2328:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.107@tcp at host 192.168.203.107 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1789761489.271145:0:2328:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.107@tcp 00000800:00000100:3.0:1789761489.271177:0:2328:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.107@tcp 00000800:00000100:3.0:1789761489.271187:0:2328:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.107@tcp 00000100:00100000:1.0:1789761489.271224:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761486/real 1789761489] req@ffffa06c091d1f80 x1876700024602496/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761511 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761489.271250:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d1f80 x1876700024602496/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761511 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761489.271262:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761489.271268:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761489.271272:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024602496:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789761489.271281:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761486/real 1789761489] req@ffffa06c091d0a80 x1876700024602624/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761511 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761489.271325:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0a80 x1876700024602624/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761511 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761489.271335:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761489.271339:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761489.271342:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024602624:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789761489.271349:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761486/real 1789761489] req@ffffa06c091d0700 x1876700024602368/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761511 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761489.271362:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0700 x1876700024602368/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761511 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761489.271371:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761489.271374:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761489.271378:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024602368:192.168.203.107@tcp:250:kworker.0 00010000:00010000:3.0:1789761489.526993:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761489.782925:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761490.550961:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761490.806899:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:1.0:1789761491.318949:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761491.318958:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789761491.318962:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761491.318980:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 925 00000100:00080000:1.0:1789761491.318990:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761491.318997:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789761491.319036:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2c700 x1876700024602752/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761491.319083:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761491.319088:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761491.319090:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761491.319095:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 925 00000100:00080000:1.0:1789761491.319099:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761491.319104:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789761491.319114:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761491.319126:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024602752:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789761491.319133:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2e300 x1876700024602880/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761491.319148:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761491.319153:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761491.319156:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761491.319161:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 925 00000100:00080000:1.0:1789761491.319185:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1789761491.319191:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00080000:1.0:1789761491.319193:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789761491.319201:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761491.319218:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2c000 x1876700024603008/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789761491.319237:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761491/real 1789761491] req@ffffa06c08e2c700 x1876700024602752/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761516 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761491.319256:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2c700 x1876700024602752/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761516 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761491.319268:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761491.319274:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789761491.319279:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024602752:192.168.203.107@tcp:250:kworker.0 00000100:00100000:0.0:1789761491.319311:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024602880:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789761491.319337:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789761491.319352:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761491/real 1789761491] req@ffffa06c08e2e300 x1876700024602880/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761516 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761491.319375:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2e300 x1876700024602880/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761516 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761491.319385:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761491.319388:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789761491.319391:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024602880:192.168.203.107@tcp:38:kworker.0 00000100:00100000:0.0:1789761491.319398:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024603008:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789761491.319423:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789761491.319436:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761491/real 1789761491] req@ffffa06c08e2c000 x1876700024603008/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761516 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761491.319450:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2c000 x1876700024603008/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761516 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761491.319459:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761491.319463:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789761491.319467:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024603008:192.168.203.107@tcp:38:kworker.0 00010000:00010000:2.0:1789761491.574948:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761491.830926:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761492.598936:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761492.854903:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761493.622928:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761493.878929:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761494.646987:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761494.902928:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761495.670947:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761495.926913:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:1.0:1789761496.438953:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761496.438960:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789761496.438964:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761496.438972:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 930 00000100:00080000:1.0:1789761496.438981:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761496.438987:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789761496.439022:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2d500 x1876700024603136/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761496.439056:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761496.439061:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761496.439063:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761496.439067:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 930 00000100:00080000:1.0:1789761496.439072:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761496.439079:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789761496.439091:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761496.439107:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2c000 x1876700024603264/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789761496.439117:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024603136:192.168.203.107@tcp:250:kworker.0 00000100:00080000:1.0:1789761496.439126:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761496.439131:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761496.439133:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761496.439138:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 930 00000100:00080000:1.0:1789761496.439141:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761496.439145:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789761496.439151:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761496.439186:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2e300 x1876700024603392/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789761496.439200:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789761496.439256:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761496/real 1789761496] req@ffffa06c08e2d500 x1876700024603136/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761521 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761496.439274:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2d500 x1876700024603136/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761521 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761496.439299:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761496.439303:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789761496.439306:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024603136:192.168.203.107@tcp:250:kworker.0 00000100:00100000:0.0:1789761496.439332:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024603264:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789761496.439351:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789761496.439360:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761496/real 1789761496] req@ffffa06c08e2c000 x1876700024603264/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761521 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761496.439368:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2c000 x1876700024603264/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761521 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761496.439374:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761496.439377:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789761496.439379:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024603264:192.168.203.107@tcp:38:kworker.0 00000100:00100000:0.0:1789761496.439383:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024603392:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789761496.439394:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789761496.439400:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761496/real 1789761496] req@ffffa06c08e2e300 x1876700024603392/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761521 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761496.439407:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2e300 x1876700024603392/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761521 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761496.439413:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761496.439415:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789761496.439417:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024603392:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789761496.695003:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761496.950891:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761497.719011:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761497.974950:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761498.742959:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761498.998954:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761499.766950:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761500.022913:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761500.790928:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761501.046982:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:1.0:1789761501.558932:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761501.558940:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789761501.558944:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761501.558953:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 935 00000100:00080000:1.0:1789761501.558963:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761501.558969:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789761501.559002:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2c700 x1876700024603520/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761501.559022:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761501.559025:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761501.559027:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761501.559030:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 935 00000100:00080000:1.0:1789761501.559033:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761501.559035:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789761501.559043:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761501.559054:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024603520:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789761501.559054:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2ca80 x1876700024603648/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761501.559070:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761501.559088:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761501.559089:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761501.559092:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 935 00000100:00080000:1.0:1789761501.559106:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761501.559110:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789761501.559114:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1789761501.559115:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761501.559128:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2ed80 x1876700024603776/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789761501.559159:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761501/real 1789761501] req@ffffa06c08e2c700 x1876700024603520/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761526 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761501.559180:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2c700 x1876700024603520/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761526 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761501.559192:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761501.559205:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789761501.559210:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024603520:192.168.203.107@tcp:250:kworker.0 00000100:00100000:0.0:1789761501.559235:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024603648:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789761501.559257:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789761501.559267:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761501/real 1789761501] req@ffffa06c08e2ca80 x1876700024603648/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761526 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761501.559276:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2ca80 x1876700024603648/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761526 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761501.559282:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761501.559284:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789761501.559287:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024603648:192.168.203.107@tcp:38:kworker.0 00000100:00100000:0.0:1789761501.559291:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024603776:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789761501.559302:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789761501.559308:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761501/real 1789761501] req@ffffa06c08e2ed80 x1876700024603776/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761526 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761501.559326:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2ed80 x1876700024603776/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761526 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761501.559336:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761501.559340:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789761501.559343:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024603776:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789761501.814946:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761502.070934:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761502.838997:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761503.094945:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761503.862963:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761504.118916:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761504.886934:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761505.142933:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761505.910954:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761506.166911:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789761506.678953:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761506.678966:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789761506.678971:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761506.678983:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 940 00000100:00080000:2.0:1789761506.678995:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761506.679002:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789761506.679043:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50000 x1876700024603904/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761506.679089:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761506.679092:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761506.679094:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761506.679097:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 940 00000100:00080000:2.0:1789761506.679100:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761506.679103:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789761506.679113:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761506.679125:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51500 x1876700024604032/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761506.679134:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761506.679138:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761506.679139:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761506.679142:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 940 00000100:00080000:2.0:1789761506.679144:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761506.679148:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789761506.679151:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761506.679158:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50e00 x1876700024604160/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789761506.679172:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024603904:192.168.203.107@tcp:250:kworker.0 00000800:00000100:0.0:1789761506.679246:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789761506.679280:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761506/real 1789761506] req@ffffa06c07e50000 x1876700024603904/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761531 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761506.679294:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50000 x1876700024603904/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761531 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761506.679303:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761506.679306:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789761506.679309:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024603904:192.168.203.107@tcp:250:kworker.0 00000100:00100000:0.0:1789761506.679326:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024604032:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789761506.679352:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789761506.679366:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761506/real 1789761506] req@ffffa06c07e51500 x1876700024604032/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761531 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761506.679380:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51500 x1876700024604032/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761531 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761506.679422:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761506.679427:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789761506.679431:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024604032:192.168.203.107@tcp:38:kworker.0 00000100:00100000:0.0:1789761506.679461:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024604160:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789761506.679480:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789761506.679489:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761506/real 1789761506] req@ffffa06c07e50e00 x1876700024604160/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761531 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761506.679497:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50e00 x1876700024604160/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761531 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761506.679502:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761506.679504:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789761506.679506:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024604160:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789761506.934974:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761507.190956:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761507.958937:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761508.214917:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761508.982917:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761509.238935:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761510.006944:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761510.262948:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761511.030970:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761511.286943:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789761511.798964:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761511.798972:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761511.798975:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761511.798983:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 945 00000100:00080000:0.0:1789761511.798992:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761511.798998:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789761511.799030:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0700 x1876700024604288/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761511.799052:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761511.799067:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761511.799068:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761511.799072:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 945 00000100:00080000:0.0:1789761511.799076:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761511.799079:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789761511.799087:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761511.799094:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024604288:192.168.203.107@tcp:250:kworker.0 00000100:00100000:0.0:1789761511.799099:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0a80 x1876700024604416/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761511.799109:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761511.799112:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761511.799114:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761511.799117:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 945 00000100:00080000:0.0:1789761511.799119:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761511.799121:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789761511.799124:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761511.799131:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d1f80 x1876700024604544/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789761511.799158:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761511.799197:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761511/real 1789761511] req@ffffa06c091d0700 x1876700024604288/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761536 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761511.799211:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0700 x1876700024604288/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761536 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761511.799219:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761511.799222:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761511.799238:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024604288:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789761511.799255:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024604416:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761511.799273:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761511.799282:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761511/real 1789761511] req@ffffa06c091d0a80 x1876700024604416/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761536 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761511.799290:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0a80 x1876700024604416/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761536 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761511.799296:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761511.799298:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761511.799300:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024604416:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789761511.799304:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024604544:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761511.799315:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761511.799322:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761511/real 1789761511] req@ffffa06c091d1f80 x1876700024604544/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761536 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761511.799333:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d1f80 x1876700024604544/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761536 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761511.799342:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761511.799345:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761511.799348:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024604544:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789761512.054960:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761512.310958:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761513.078962:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761513.334978:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761514.102973:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761514.358955:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761515.126943:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761515.382952:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761516.151024:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761516.406991:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789761516.918997:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761516.919009:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789761516.919014:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761516.919025:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 950 00000100:00080000:2.0:1789761516.919053:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761516.919062:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789761516.919106:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e53b80 x1876700024604672/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761516.919136:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761516.919141:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761516.919144:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761516.919149:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 950 00000100:00080000:2.0:1789761516.919154:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761516.919170:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789761516.919181:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e53b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024604672:192.168.203.107@tcp:250:kworker.0 00000008:00100000:2.0:1789761516.919182:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761516.919202:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50380 x1876700024604800/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761516.919219:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761516.919223:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761516.919225:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761516.919231:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 950 00000100:00080000:2.0:1789761516.919236:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1789761516.919237:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00080000:2.0:1789761516.919245:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789761516.919253:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761516.919270:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51f80 x1876700024604928/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789761516.919278:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761516/real 1789761516] req@ffffa06c07e53b80 x1876700024604672/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761541 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761516.919295:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e53b80 x1876700024604672/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761541 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761516.919304:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761516.919307:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761516.919310:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e53b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024604672:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789761516.919344:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024604800:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761516.919378:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761516.919392:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761516/real 1789761516] req@ffffa06c07e50380 x1876700024604800/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761541 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761516.919405:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50380 x1876700024604800/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761541 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761516.919415:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761516.919418:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761516.919422:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024604800:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789761516.919429:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024604928:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761516.919456:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761516.919471:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761516/real 1789761516] req@ffffa06c07e51f80 x1876700024604928/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761541 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761516.919480:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51f80 x1876700024604928/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761541 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761516.919485:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761516.919487:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761516.919490:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024604928:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789761517.175006:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761517.430921:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761518.199034:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761518.454952:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761519.222922:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761519.478956:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761520.247006:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761520.502942:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761521.270954:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761521.526918:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:1.0:1789761522.038952:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761522.038960:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789761522.038964:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761522.038983:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 955 00000100:00080000:1.0:1789761522.038991:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761522.038997:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789761522.039049:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2d180 x1876700024605056/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761522.039075:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761522.039079:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761522.039081:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761522.039085:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 955 00000100:00080000:1.0:1789761522.039090:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761522.039109:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789761522.039120:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761522.039121:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024605056:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789761522.039144:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2ed80 x1876700024605184/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761522.039160:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761522.039164:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761522.039166:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761522.039170:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 955 00000100:00080000:1.0:1789761522.039176:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1789761522.039180:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00080000:1.0:1789761522.039184:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789761522.039192:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761522.039208:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2ca80 x1876700024605312/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789761522.039218:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761522/real 1789761522] req@ffffa06c08e2d180 x1876700024605056/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761547 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761522.039233:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2d180 x1876700024605056/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761547 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761522.039251:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761522.039255:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789761522.039257:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024605056:192.168.203.107@tcp:250:kworker.0 00000100:00100000:0.0:1789761522.039275:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024605184:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789761522.039310:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789761522.039320:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761522/real 1789761522] req@ffffa06c08e2ed80 x1876700024605184/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761547 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761522.039329:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2ed80 x1876700024605184/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761547 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761522.039335:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761522.039337:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789761522.039339:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024605184:192.168.203.107@tcp:38:kworker.0 00000100:00100000:0.0:1789761522.039343:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024605312:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789761522.039354:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789761522.039361:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761522/real 1789761522] req@ffffa06c08e2ca80 x1876700024605312/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761547 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761522.039368:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2ca80 x1876700024605312/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761547 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761522.039373:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761522.039375:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789761522.039377:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024605312:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789761522.294964:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761522.550895:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761523.318964:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761523.574927:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761524.342957:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761524.598927:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761525.366934:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761525.622898:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761526.391017:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761526.646931:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:1.0:1789761527.158931:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761527.158940:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789761527.158943:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761527.158952:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 960 00000100:00080000:1.0:1789761527.158960:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761527.158979:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789761527.159008:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2c700 x1876700024605440/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761527.159030:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761527.159033:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761527.159035:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761527.159038:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 960 00000100:00080000:1.0:1789761527.159041:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761527.159044:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789761527.159051:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761527.159063:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2e300 x1876700024605568/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761527.159074:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761527.159077:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761527.159080:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1789761527.159081:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024605440:192.168.203.107@tcp:250:kworker.0 00000100:00080000:1.0:1789761527.159088:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 960 00000100:00080000:1.0:1789761527.159091:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761527.159094:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789761527.159098:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761527.159108:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2c000 x1876700024605696/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789761527.159139:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789761527.159174:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761527/real 1789761527] req@ffffa06c08e2c700 x1876700024605440/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761552 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761527.159188:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2c700 x1876700024605440/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761552 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761527.159198:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761527.159201:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789761527.159205:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024605440:192.168.203.107@tcp:250:kworker.0 00000100:00100000:0.0:1789761527.159226:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024605568:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789761527.159240:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789761527.159265:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761527/real 1789761527] req@ffffa06c08e2e300 x1876700024605568/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761552 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761527.159279:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2e300 x1876700024605568/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761552 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761527.159288:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761527.159292:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789761527.159296:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024605568:192.168.203.107@tcp:38:kworker.0 00000100:00100000:0.0:1789761527.159303:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024605696:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789761527.159323:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789761527.159333:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761527/real 1789761527] req@ffffa06c08e2c000 x1876700024605696/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761552 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761527.159349:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2c000 x1876700024605696/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761552 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761527.159358:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761527.159361:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789761527.159365:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024605696:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789761527.414941:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761527.670899:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761528.438939:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761528.694974:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761529.462989:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761529.718938:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761530.486979:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761530.742974:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761531.510951:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761531.766918:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:1.0:1789761532.278930:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761532.278942:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789761532.278948:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761532.278961:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 966 00000100:00080000:1.0:1789761532.278976:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761532.278986:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789761532.279034:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2d500 x1876700024605824/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761532.279062:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761532.279065:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761532.279067:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761532.279070:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 966 00000100:00080000:1.0:1789761532.279073:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761532.279077:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789761532.279087:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761532.279101:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2c000 x1876700024605952/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789761532.279108:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024605824:192.168.203.107@tcp:250:kworker.0 00000100:00080000:1.0:1789761532.279115:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761532.279119:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761532.279120:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761532.279124:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 966 00000100:00080000:1.0:1789761532.279134:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761532.279138:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789761532.279162:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761532.279173:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2e300 x1876700024606080/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789761532.279181:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789761532.279242:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761532/real 1789761532] req@ffffa06c08e2d500 x1876700024605824/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761557 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761532.279258:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2d500 x1876700024605824/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761557 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761532.279266:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761532.279269:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789761532.279273:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024605824:192.168.203.107@tcp:250:kworker.0 00000100:00100000:0.0:1789761532.279293:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024605952:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789761532.279311:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789761532.279319:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761532/real 1789761532] req@ffffa06c08e2c000 x1876700024605952/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761557 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761532.279328:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2c000 x1876700024605952/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761557 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761532.279356:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761532.279360:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789761532.279362:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024605952:192.168.203.107@tcp:38:kworker.0 00000100:00100000:0.0:1789761532.279367:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024606080:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789761532.279382:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789761532.279390:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761532/real 1789761532] req@ffffa06c08e2e300 x1876700024606080/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761557 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761532.279397:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2e300 x1876700024606080/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761557 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761532.279402:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761532.279405:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789761532.279407:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024606080:192.168.203.107@tcp:38:kworker.0 00010000:00010000:2.0:1789761532.535017:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761532.790966:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761533.558954:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761533.814937:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761534.583000:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761534.838961:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761535.607046:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761535.862945:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761536.630947:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761536.886939:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789761537.399002:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761537.399012:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789761537.399017:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761537.399027:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 971 00000100:00080000:2.0:1789761537.399040:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761537.399047:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789761537.399094:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e52680 x1876700024606208/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761537.399129:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761537.399134:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761537.399135:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761537.399140:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 971 00000100:00080000:2.0:1789761537.399143:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761537.399147:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789761537.399157:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761537.399186:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e53800 x1876700024606336/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789761537.399193:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e52680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024606208:192.168.203.107@tcp:250:kworker.0 00000100:00080000:2.0:1789761537.399199:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761537.399204:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761537.399205:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761537.399209:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 971 00000100:00080000:2.0:1789761537.399212:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761537.399237:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789761537.399242:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761537.399252:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51c00 x1876700024606464/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789761537.399278:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789761537.399324:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761537/real 1789761537] req@ffffa06c07e52680 x1876700024606208/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761562 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761537.399339:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e52680 x1876700024606208/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761562 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761537.399348:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761537.399352:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789761537.399378:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e52680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024606208:192.168.203.107@tcp:250:kworker.0 00000100:00100000:0.0:1789761537.399408:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e53800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024606336:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789761537.399434:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789761537.399445:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761537/real 1789761537] req@ffffa06c07e53800 x1876700024606336/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761562 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761537.399454:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e53800 x1876700024606336/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761562 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761537.399460:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761537.399464:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789761537.399467:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e53800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024606336:192.168.203.107@tcp:38:kworker.0 00000100:00100000:0.0:1789761537.399474:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024606464:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789761537.399494:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789761537.399506:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761537/real 1789761537] req@ffffa06c07e51c00 x1876700024606464/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761562 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761537.399519:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51c00 x1876700024606464/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761562 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761537.399528:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761537.399532:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789761537.399535:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024606464:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789761537.655045:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761537.910965:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789761538.678951:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761538.934934:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761539.702961:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761539.958939:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761540.726945:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761540.982936:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789761541.750971:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761542.006944:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:1.0:1789761542.519005:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761542.519013:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789761542.519016:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761542.519024:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 976 00000100:00080000:1.0:1789761542.519039:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761542.519047:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789761542.519085:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2c700 x1876700024606592/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761542.519112:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761542.519115:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761542.519117:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761542.519120:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 976 00000100:00080000:1.0:1789761542.519123:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761542.519152:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789761542.519161:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761542.519172:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2ca80 x1876700024606720/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761542.519182:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1789761542.519185:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024606592:192.168.203.107@tcp:250:kworker.0 00000100:00080000:1.0:1789761542.519185:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761542.519189:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761542.519196:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 976 00000100:00080000:1.0:1789761542.519200:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761542.519203:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789761542.519206:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761542.519215:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2ed80 x1876700024606848/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789761542.519254:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789761542.519296:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761542/real 1789761542] req@ffffa06c08e2c700 x1876700024606592/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761567 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761542.519309:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2c700 x1876700024606592/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761567 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761542.519317:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761542.519320:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789761542.519323:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024606592:192.168.203.107@tcp:250:kworker.0 00000100:00100000:0.0:1789761542.519341:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024606720:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789761542.519354:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789761542.519362:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761542/real 1789761542] req@ffffa06c08e2ca80 x1876700024606720/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761567 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761542.519370:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2ca80 x1876700024606720/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761567 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761542.519376:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761542.519378:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789761542.519380:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024606720:192.168.203.107@tcp:38:kworker.0 00000100:00100000:0.0:1789761542.519384:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024606848:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789761542.519394:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789761542.519400:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761542/real 1789761542] req@ffffa06c08e2ed80 x1876700024606848/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761567 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761542.519418:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2ed80 x1876700024606848/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761567 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761542.519424:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761542.519426:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789761542.519428:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024606848:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789761542.775004:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761543.030930:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761543.798961:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761544.054953:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761544.822954:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761545.078907:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761545.846961:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761546.102933:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761546.870941:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761547.126906:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:1.0:1789761547.638925:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761547.638934:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789761547.638938:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761547.638947:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 981 00000100:00080000:1.0:1789761547.638957:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761547.638965:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789761547.639021:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2d180 x1876700024606976/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761547.639048:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761547.639052:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761547.639054:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761547.639059:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 981 00000100:00080000:1.0:1789761547.639065:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761547.639070:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789761547.639080:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761547.639087:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024606976:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789761547.639107:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2ed80 x1876700024607104/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761547.639119:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761547.639122:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761547.639124:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761547.639126:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 981 00000100:00080000:1.0:1789761547.639142:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761547.639145:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789761547.639149:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1789761547.639152:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761547.639163:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2ca80 x1876700024607232/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789761547.639187:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761547/real 1789761547] req@ffffa06c08e2d180 x1876700024606976/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761572 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761547.639200:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2d180 x1876700024606976/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761572 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761547.639208:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761547.639212:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789761547.639217:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024606976:192.168.203.107@tcp:250:kworker.0 00000100:00100000:0.0:1789761547.639239:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024607104:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789761547.639262:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789761547.639275:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761547/real 1789761547] req@ffffa06c08e2ed80 x1876700024607104/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761572 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761547.639284:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2ed80 x1876700024607104/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761572 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761547.639299:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761547.639302:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789761547.639304:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024607104:192.168.203.107@tcp:38:kworker.0 00000100:00100000:0.0:1789761547.639308:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024607232:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789761547.639325:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789761547.639334:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761547/real 1789761547] req@ffffa06c08e2ca80 x1876700024607232/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761572 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761547.639341:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2ca80 x1876700024607232/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761572 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761547.639347:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761547.639349:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789761547.639351:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024607232:192.168.203.107@tcp:38:kworker.0 00010000:00010000:1.0:1789761547.895000:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761548.150946:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761548.918964:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761549.174947:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761549.942966:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761550.198928:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761550.967025:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761551.222927:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761551.990989:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761552.246974:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:1.0:1789761552.758972:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761552.758985:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789761552.758990:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761552.759001:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 986 00000100:00080000:1.0:1789761552.759014:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761552.759021:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789761552.759074:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2c700 x1876700024607360/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761552.759101:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761552.759104:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761552.759105:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761552.759109:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 986 00000100:00080000:1.0:1789761552.759111:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761552.759114:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789761552.759124:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761552.759134:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2e300 x1876700024607488/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761552.759144:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761552.759147:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761552.759148:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761552.759153:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 986 00000100:00080000:1.0:1789761552.759156:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761552.759158:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789761552.759168:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024607360:192.168.203.107@tcp:250:kworker.0 00000008:00100000:1.0:1789761552.759180:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761552.759191:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2c000 x1876700024607616/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789761552.759314:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024607488:192.168.203.107@tcp:38:kworker.0 00000100:00100000:0.0:1789761552.759345:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024607616:192.168.203.107@tcp:38:kworker.0 00010000:00010000:2.0:1789761553.015013:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761553.270985:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761554.038948:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761554.294929:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761555.062949:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761555.318936:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000400:00000100:0.0:1789761555.831248:0:2329:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.107:988 00000400:00000100:0.0:1789761555.831294:0:2329:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.107@tcp at host 192.168.203.107 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1789761555.831321:0:2329:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.107@tcp 00000800:00000100:0.0:1789761555.831367:0:2329:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.107@tcp 00000800:00000100:0.0:1789761555.831379:0:2329:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.107@tcp 00000100:00100000:1.0:1789761555.831399:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761552/real 1789761555] req@ffffa06c08e2e300 x1876700024607488/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761577 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761555.831426:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2e300 x1876700024607488/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761577 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761555.831438:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761555.831442:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761555.831446:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024607488:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789761555.831454:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761552/real 1789761555] req@ffffa06c08e2c000 x1876700024607616/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761577 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761555.831467:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2c000 x1876700024607616/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761577 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761555.831475:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761555.831479:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761555.831482:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024607616:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789761555.831488:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761552/real 1789761555] req@ffffa06c08e2c700 x1876700024607360/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761577 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761555.831517:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2c700 x1876700024607360/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761577 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761555.831526:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761555.831530:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761555.831533:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024607360:192.168.203.107@tcp:250:kworker.0 00010000:00010000:0.0:1789761556.086959:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761556.342912:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761557.110957:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761557.366952:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789761557.878957:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761557.878969:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789761557.878972:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761557.878981:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 991 00000100:00080000:2.0:1789761557.878989:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761557.878995:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789761557.879039:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50a80 x1876700024607744/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761557.879065:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761557.879076:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761557.879078:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761557.879087:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 991 00000100:00080000:2.0:1789761557.879092:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761557.879097:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789761557.879107:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761557.879109:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024607744:192.168.203.107@tcp:250:kworker.0 00000100:00100000:2.0:1789761557.879126:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50700 x1876700024607872/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761557.879155:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761557.879158:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761557.879160:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761557.879165:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 991 00000100:00080000:2.0:1789761557.879171:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761557.879175:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000800:00000100:1.0:1789761557.879182:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000008:00100000:2.0:1789761557.879182:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761557.879201:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e53480 x1876700024608000/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789761557.879231:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761557/real 1789761557] req@ffffa06c07e50a80 x1876700024607744/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761582 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761557.879250:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50a80 x1876700024607744/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761582 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761557.879287:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761557.879291:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761557.879295:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024607744:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789761557.879321:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024607872:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761557.879346:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761557.879360:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761557/real 1789761557] req@ffffa06c07e50700 x1876700024607872/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761582 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761557.879373:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50700 x1876700024607872/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761582 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761557.879390:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761557.879395:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761557.879399:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024607872:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789761557.879406:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e53480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024608000:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761557.879430:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761557.879443:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761557/real 1789761557] req@ffffa06c07e53480 x1876700024608000/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761582 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761557.879454:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e53480 x1876700024608000/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761582 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761557.879464:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761557.879468:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761557.879471:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e53480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024608000:192.168.203.107@tcp:38:kworker.0 00010000:00010000:2.0:1789761558.134941:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761558.390936:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789761559.159026:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761559.414943:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761560.182944:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761560.438934:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761561.206971:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761561.462911:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761562.230943:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761562.486916:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789761562.998954:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761562.998961:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761562.998964:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761562.998973:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 996 00000100:00080000:0.0:1789761562.998999:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761562.999006:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789761562.999036:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0380 x1876700024608128/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761562.999058:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761562.999061:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761562.999063:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761562.999067:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 996 00000100:00080000:0.0:1789761562.999070:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761562.999074:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789761562.999082:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761562.999090:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024608128:192.168.203.107@tcp:250:kworker.0 00000100:00100000:0.0:1789761562.999096:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d2680 x1876700024608256/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761562.999107:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761562.999110:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761562.999112:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761562.999115:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 996 00000100:00080000:0.0:1789761562.999119:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761562.999130:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789761562.999134:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761562.999143:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d3100 x1876700024608384/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789761562.999152:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789761562.999186:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761562/real 1789761562] req@ffffa06c091d0380 x1876700024608128/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761587 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761562.999199:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0380 x1876700024608128/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761587 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761562.999208:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761562.999211:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789761562.999214:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024608128:192.168.203.107@tcp:250:kworker.0 00000100:00100000:2.0:1789761562.999232:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024608256:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789761562.999265:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789761562.999286:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761562/real 1789761562] req@ffffa06c091d2680 x1876700024608256/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761587 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761562.999296:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d2680 x1876700024608256/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761587 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761562.999302:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761562.999305:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789761562.999308:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024608256:192.168.203.107@tcp:38:kworker.0 00000100:00100000:2.0:1789761562.999312:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024608384:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789761562.999324:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789761562.999331:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761562/real 1789761562] req@ffffa06c091d3100 x1876700024608384/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761587 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761562.999338:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d3100 x1876700024608384/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761587 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761562.999343:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761562.999345:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789761562.999347:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024608384:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789761563.254965:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761563.510934:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761564.278966:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761564.534926:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789761565.302997:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761565.558957:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761566.327002:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761566.582925:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761567.350964:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761567.606935:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789761568.118985:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761568.118994:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789761568.118997:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761568.119006:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1001 00000100:00080000:2.0:1789761568.119015:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761568.119021:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789761568.119064:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e52a00 x1876700024608512/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761568.119088:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761568.119092:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761568.119093:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761568.119096:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1001 00000100:00080000:2.0:1789761568.119099:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761568.119102:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789761568.119109:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761568.119118:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e53100 x1876700024608640/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761568.119127:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1789761568.119129:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e52a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024608512:192.168.203.107@tcp:250:kworker.0 00000100:00080000:2.0:1789761568.119130:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761568.119134:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761568.119139:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1001 00000100:00080000:2.0:1789761568.119141:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761568.119143:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789761568.119146:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761568.119155:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e52d80 x1876700024608768/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789761568.119186:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761568.119225:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761568/real 1789761568] req@ffffa06c07e52a00 x1876700024608512/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761593 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761568.119243:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e52a00 x1876700024608512/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761593 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761568.119255:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761568.119260:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761568.119264:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e52a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024608512:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789761568.119288:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e53100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024608640:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761568.119309:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761568.119321:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761568/real 1789761568] req@ffffa06c07e53100 x1876700024608640/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761593 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761568.119335:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e53100 x1876700024608640/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761593 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761568.119344:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761568.119348:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761568.119351:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e53100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024608640:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789761568.119358:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e52d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024608768:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761568.119391:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761568.119403:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761568/real 1789761568] req@ffffa06c07e52d80 x1876700024608768/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761593 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761568.119424:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e52d80 x1876700024608768/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761593 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761568.119430:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761568.119432:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761568.119434:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e52d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024608768:192.168.203.107@tcp:38:kworker.0 00010000:00010000:0.0:1789761568.374972:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761568.630916:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761569.398982:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761569.654985:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761570.422980:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761570.678939:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789761571.446983:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761571.702949:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761572.470976:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761572.726930:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:1.0:1789761573.238950:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761573.238957:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789761573.238961:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761573.238970:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1007 00000100:00080000:1.0:1789761573.238980:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761573.238987:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789761573.239025:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2c700 x1876700024608896/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761573.239054:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761573.239059:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761573.239061:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761573.239066:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1007 00000100:00080000:1.0:1789761573.239083:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761573.239089:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789761573.239093:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024608896:192.168.203.107@tcp:250:kworker.0 00000008:00100000:1.0:1789761573.239099:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761573.239115:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2c000 x1876700024609024/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761573.239125:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761573.239128:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761573.239130:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761573.239133:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1007 00000100:00080000:1.0:1789761573.239136:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761573.239139:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789761573.239142:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1789761573.239149:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761573.239153:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2e300 x1876700024609152/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789761573.239199:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761573/real 1789761573] req@ffffa06c08e2c700 x1876700024608896/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761598 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761573.239220:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2c700 x1876700024608896/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761598 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761573.239232:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761573.239236:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789761573.239240:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024608896:192.168.203.107@tcp:250:kworker.0 00000100:00100000:2.0:1789761573.239265:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024609024:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789761573.239292:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789761573.239308:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761573/real 1789761573] req@ffffa06c08e2c000 x1876700024609024/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761598 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761573.239322:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2c000 x1876700024609024/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761598 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761573.239328:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761573.239330:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789761573.239333:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024609024:192.168.203.107@tcp:38:kworker.0 00000100:00100000:2.0:1789761573.239338:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024609152:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789761573.239354:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789761573.239361:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761573/real 1789761573] req@ffffa06c08e2e300 x1876700024609152/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761598 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761573.239369:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2e300 x1876700024609152/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761598 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761573.239384:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761573.239386:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789761573.239389:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024609152:192.168.203.107@tcp:38:kworker.0 00010000:00010000:2.0:1789761573.495007:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761573.751008:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789761574.519048:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761574.775025:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789761575.542964:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761575.798977:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761576.566966:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761576.822988:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761577.590979:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761577.847028:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789761578.358958:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761578.358966:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761578.358970:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761578.358977:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1012 00000100:00080000:0.0:1789761578.358986:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761578.358992:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789761578.359022:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d1500 x1876700024609280/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761578.359045:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761578.359048:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761578.359049:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761578.359052:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1012 00000100:00080000:0.0:1789761578.359054:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761578.359056:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789761578.359064:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761578.359073:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d2a00 x1876700024609408/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761578.359095:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761578.359097:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761578.359100:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761578.359103:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1012 00000100:00080000:0.0:1789761578.359105:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761578.359108:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789761578.359111:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761578.359122:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0e00 x1876700024609536/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789761578.359122:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024609280:192.168.203.107@tcp:250:kworker.0 00000800:00000100:2.0:1789761578.359231:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789761578.359282:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761578/real 1789761578] req@ffffa06c091d1500 x1876700024609280/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761603 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761578.359298:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d1500 x1876700024609280/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761603 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761578.359307:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761578.359311:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789761578.359314:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024609280:192.168.203.107@tcp:250:kworker.0 00000100:00100000:2.0:1789761578.359321:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024609408:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789761578.359339:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789761578.359348:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761578/real 1789761578] req@ffffa06c091d2a00 x1876700024609408/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761603 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761578.359356:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d2a00 x1876700024609408/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761603 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761578.359378:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761578.359381:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789761578.359383:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024609408:192.168.203.107@tcp:38:kworker.0 00000100:00100000:2.0:1789761578.359408:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024609536:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789761578.359423:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789761578.359430:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761578/real 1789761578] req@ffffa06c091d0e00 x1876700024609536/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761603 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761578.359438:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0e00 x1876700024609536/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761603 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761578.359443:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761578.359445:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789761578.359447:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024609536:192.168.203.107@tcp:38:kworker.0 00010000:00010000:0.0:1789761578.615024:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761578.870956:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789761579.638962:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761579.894952:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761580.663047:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761580.918983:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761581.686971:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761581.942950:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789761582.711040:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761582.966981:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789761583.478971:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761583.478986:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761583.478992:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761583.479006:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1017 00000100:00080000:0.0:1789761583.479020:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761583.479030:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789761583.479079:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d3b80 x1876700024609664/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761583.479108:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761583.479111:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761583.479113:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761583.479133:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1017 00000100:00080000:0.0:1789761583.479137:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761583.479140:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789761583.479156:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761583.479167:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d2300 x1876700024609792/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761583.479177:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761583.479180:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761583.479182:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761583.479187:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1017 00000100:00080000:0.0:1789761583.479190:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1789761583.479197:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024609664:192.168.203.107@tcp:250:kworker.0 00000100:00080000:0.0:1789761583.479199:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789761583.479210:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761583.479234:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0000 x1876700024609920/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789761583.479285:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789761583.479331:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761583/real 1789761583] req@ffffa06c091d3b80 x1876700024609664/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761608 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761583.479347:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d3b80 x1876700024609664/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761608 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761583.479356:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761583.479359:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789761583.479370:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024609664:192.168.203.107@tcp:250:kworker.0 00000100:00100000:2.0:1789761583.479376:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024609792:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789761583.479395:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789761583.479404:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761583/real 1789761583] req@ffffa06c091d2300 x1876700024609792/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761608 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761583.479419:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d2300 x1876700024609792/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761608 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761583.479428:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761583.479430:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789761583.479433:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024609792:192.168.203.107@tcp:38:kworker.0 00000100:00100000:2.0:1789761583.479458:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024609920:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789761583.479471:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789761583.479478:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761583/real 1789761583] req@ffffa06c091d0000 x1876700024609920/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761608 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761583.479485:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0000 x1876700024609920/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761608 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761583.479491:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761583.479493:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789761583.479495:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024609920:192.168.203.107@tcp:38:kworker.0 00010000:00010000:2.0:1789761583.734977:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761583.990940:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761584.759048:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761585.014982:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761585.783103:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761586.038938:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761586.806988:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761587.062966:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761587.831032:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761588.086937:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789761588.598978:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761588.598991:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761588.598996:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761588.599010:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1022 00000100:00080000:0.0:1789761588.599024:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761588.599031:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789761588.599076:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0000 x1876700024610048/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761588.599110:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761588.599135:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761588.599137:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761588.599142:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1022 00000100:00080000:0.0:1789761588.599146:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1789761588.599151:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024610048:192.168.203.107@tcp:250:kworker.0 00000100:00080000:0.0:1789761588.599158:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789761588.599172:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761588.599190:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d2300 x1876700024610176/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761588.599206:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761588.599211:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761588.599213:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761588.599218:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1022 00000100:00080000:0.0:1789761588.599223:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1789761588.599228:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00080000:0.0:1789761588.599232:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789761588.599239:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761588.599255:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d3b80 x1876700024610304/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789761588.599277:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761588/real 1789761588] req@ffffa06c091d0000 x1876700024610048/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761613 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761588.599295:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0000 x1876700024610048/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761613 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761588.599305:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761588.599308:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789761588.599312:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024610048:192.168.203.107@tcp:250:kworker.0 00000100:00100000:2.0:1789761588.599334:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024610176:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789761588.599350:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789761588.599359:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761588/real 1789761588] req@ffffa06c091d2300 x1876700024610176/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761613 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761588.599368:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d2300 x1876700024610176/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761613 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761588.599374:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761588.599377:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789761588.599379:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024610176:192.168.203.107@tcp:38:kworker.0 00000100:00100000:2.0:1789761588.599383:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024610304:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789761588.599394:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789761588.599400:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761588/real 1789761588] req@ffffa06c091d3b80 x1876700024610304/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761613 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761588.599443:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d3b80 x1876700024610304/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761613 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761588.599450:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761588.599455:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789761588.599459:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024610304:192.168.203.107@tcp:38:kworker.0 00010000:00010000:2.0:1789761588.855000:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761589.110951:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761589.879042:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761590.134963:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761590.902990:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761591.158982:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761591.927056:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761592.182944:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761592.951005:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761593.206980:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789761593.718976:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761593.718991:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761593.718996:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761593.719007:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1027 00000100:00080000:0.0:1789761593.719019:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761593.719027:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789761593.719073:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0e00 x1876700024610432/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761593.719166:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761593.719169:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761593.719171:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761593.719175:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1027 00000100:00080000:0.0:1789761593.719179:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761593.719182:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789761593.719192:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761593.719204:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d2a00 x1876700024610560/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761593.719214:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761593.719217:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761593.719221:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1789761593.719222:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024610432:192.168.203.107@tcp:250:kworker.0 00000100:00080000:0.0:1789761593.719229:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1027 00000100:00080000:0.0:1789761593.719235:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761593.719238:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789761593.719243:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761593.719255:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d1500 x1876700024610688/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789761593.719308:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789761593.719360:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761593/real 1789761593] req@ffffa06c091d0e00 x1876700024610432/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761618 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761593.719375:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0e00 x1876700024610432/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761618 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761593.719385:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761593.719388:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789761593.719391:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024610432:192.168.203.107@tcp:250:kworker.0 00000100:00100000:2.0:1789761593.719416:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024610560:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789761593.719433:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789761593.719442:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761593/real 1789761593] req@ffffa06c091d2a00 x1876700024610560/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761618 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761593.719450:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d2a00 x1876700024610560/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761618 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761593.719462:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761593.719465:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789761593.719468:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024610560:192.168.203.107@tcp:38:kworker.0 00000100:00100000:2.0:1789761593.719472:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024610688:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789761593.719487:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789761593.719494:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761593/real 1789761593] req@ffffa06c091d1500 x1876700024610688/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761618 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761593.719501:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d1500 x1876700024610688/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761618 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761593.719507:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761593.719509:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789761593.719510:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024610688:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789761593.974952:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761594.230986:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761594.998974:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761595.254968:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761596.022996:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761596.278951:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761597.046952:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761597.302935:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761598.070978:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761598.326951:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789761598.838908:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761598.838916:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789761598.838919:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761598.838927:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1032 00000100:00080000:2.0:1789761598.838936:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761598.838942:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789761598.838973:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e52d80 x1876700024610816/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761598.839020:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761598.839025:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761598.839027:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761598.839032:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1032 00000100:00080000:2.0:1789761598.839037:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761598.839042:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789761598.839053:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761598.839065:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e52d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024610816:192.168.203.107@tcp:250:kworker.0 00000100:00100000:2.0:1789761598.839072:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e53100 x1876700024610944/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761598.839087:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761598.839091:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761598.839094:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761598.839098:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1032 00000100:00080000:2.0:1789761598.839103:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761598.839107:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789761598.839111:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1789761598.839120:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789761598.839124:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e52a00 x1876700024611072/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789761598.839156:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761598/real 1789761598] req@ffffa06c07e52d80 x1876700024610816/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761623 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761598.839171:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e52d80 x1876700024610816/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761623 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761598.839180:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761598.839183:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761598.839187:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e52d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024610816:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789761598.839207:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e53100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024610944:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761598.839237:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761598.839247:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761598/real 1789761598] req@ffffa06c07e53100 x1876700024610944/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761623 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761598.839257:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e53100 x1876700024610944/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761623 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761598.839273:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761598.839276:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761598.839278:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e53100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024610944:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789761598.839284:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e52a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024611072:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761598.839315:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761598.839330:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761598/real 1789761598] req@ffffa06c07e52a00 x1876700024611072/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761623 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761598.839347:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e52a00 x1876700024611072/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761623 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761598.839357:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761598.839363:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761598.839369:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e52a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024611072:192.168.203.107@tcp:38:kworker.0 00010000:00010000:1.0:1789761599.094954:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761599.350929:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761600.118948:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761600.374933:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761601.142932:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761601.398932:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761602.166957:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761602.422938:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761603.190939:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761603.446924:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789761603.958939:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761603.958949:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789761603.958954:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761603.958985:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1037 00000100:00080000:2.0:1789761603.958997:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761603.959004:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789761603.959041:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e53480 x1876700024611200/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761603.959068:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761603.959071:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761603.959072:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761603.959076:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1037 00000100:00080000:2.0:1789761603.959078:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761603.959080:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789761603.959089:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761603.959098:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50700 x1876700024611328/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761603.959107:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761603.959110:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761603.959112:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761603.959114:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1037 00000100:00080000:2.0:1789761603.959116:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761603.959118:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789761603.959122:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761603.959129:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50a80 x1876700024611456/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789761603.959139:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e53480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024611200:192.168.203.107@tcp:250:kworker.0 00000800:00000100:1.0:1789761603.959230:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761603.959287:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761603/real 1789761603] req@ffffa06c07e53480 x1876700024611200/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761628 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761603.959310:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e53480 x1876700024611200/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761628 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761603.959324:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761603.959329:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761603.959333:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e53480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024611200:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789761603.959341:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024611328:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761603.959363:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761603.959373:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761603/real 1789761603] req@ffffa06c07e50700 x1876700024611328/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761628 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761603.959381:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50700 x1876700024611328/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761628 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761603.959387:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761603.959390:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761603.959392:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024611328:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789761603.959415:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024611456:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761603.959427:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761603.959443:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761603/real 1789761603] req@ffffa06c07e50a80 x1876700024611456/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761628 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761603.959451:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50a80 x1876700024611456/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761628 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761603.959456:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761603.959458:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761603.959461:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024611456:192.168.203.107@tcp:38:kworker.0 00010000:00010000:1.0:1789761604.214999:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761604.470964:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761605.239009:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761605.494932:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789761606.263032:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761606.518932:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789761607.287023:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761607.542929:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761608.311015:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761608.566917:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789761609.078932:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761609.078940:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789761609.078944:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761609.078952:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1042 00000100:00080000:2.0:1789761609.078961:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761609.078967:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789761609.078998:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51c00 x1876700024611584/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761609.079022:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761609.079024:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761609.079026:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761609.079029:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1042 00000100:00080000:2.0:1789761609.079031:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761609.079033:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789761609.079040:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761609.079048:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e53800 x1876700024611712/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761609.079057:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761609.079060:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761609.079061:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761609.079064:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1042 00000100:00080000:2.0:1789761609.079066:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761609.079072:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789761609.079074:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024611584:192.168.203.107@tcp:250:kworker.0 00000008:00100000:2.0:1789761609.079089:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761609.079099:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e52680 x1876700024611840/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789761609.079130:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761609.079161:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761609/real 1789761609] req@ffffa06c07e51c00 x1876700024611584/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761634 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761609.079174:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51c00 x1876700024611584/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761634 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761609.079182:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761609.079185:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761609.079188:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024611584:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789761609.079202:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e53800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024611712:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761609.079218:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761609.079227:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761609/real 1789761609] req@ffffa06c07e53800 x1876700024611712/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761634 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761609.079238:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e53800 x1876700024611712/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761634 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761609.079247:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761609.079252:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761609.079255:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e53800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024611712:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789761609.079281:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e52680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024611840:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761609.079304:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761609.079328:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761609/real 1789761609] req@ffffa06c07e52680 x1876700024611840/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761634 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761609.079345:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e52680 x1876700024611840/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761634 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761609.079351:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761609.079354:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761609.079356:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e52680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024611840:192.168.203.107@tcp:38:kworker.0 00010000:00010000:0.0:1789761609.335003:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761609.590923:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761610.359006:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761610.614959:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761611.383006:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761611.638985:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789761612.407034:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761612.662995:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761613.431023:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761613.686969:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789761614.198909:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761614.198917:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789761614.198920:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761614.198928:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1048 00000100:00080000:2.0:1789761614.198937:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761614.198943:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789761614.198979:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51f80 x1876700024611968/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761614.199008:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761614.199012:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761614.199018:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761614.199023:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1048 00000100:00080000:2.0:1789761614.199027:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761614.199031:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789761614.199042:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761614.199058:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50380 x1876700024612096/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789761614.199060:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024611968:192.168.203.107@tcp:250:kworker.0 00000100:00080000:2.0:1789761614.199076:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761614.199079:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761614.199081:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761614.199085:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1048 00000100:00080000:2.0:1789761614.199101:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761614.199105:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789761614.199112:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1789761614.199117:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789761614.199128:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e53b80 x1876700024612224/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789761614.199155:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761614/real 1789761614] req@ffffa06c07e51f80 x1876700024611968/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761639 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761614.199169:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51f80 x1876700024611968/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761639 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761614.199178:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761614.199181:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761614.199183:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024611968:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789761614.199201:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024612096:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761614.199216:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761614.199224:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761614/real 1789761614] req@ffffa06c07e50380 x1876700024612096/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761639 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761614.199232:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50380 x1876700024612096/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761639 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761614.199238:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761614.199240:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761614.199242:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024612096:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789761614.199246:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e53b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024612224:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761614.199276:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761614.199283:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761614/real 1789761614] req@ffffa06c07e53b80 x1876700024612224/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761639 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761614.199291:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e53b80 x1876700024612224/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761639 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761614.199296:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761614.199298:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761614.199300:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e53b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024612224:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789761614.454982:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761614.710971:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761615.479023:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761615.734931:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761616.502990:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761616.758937:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761617.526970:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761617.783001:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761618.551107:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761618.806996:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:3.0:1789761619.318918:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761619.318929:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789761619.318933:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761619.318943:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1053 00000100:00080000:3.0:1789761619.318963:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761619.318973:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:3.0:1789761619.319023:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872f480 x1876700024612352/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761619.319048:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761619.319068:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761619.319070:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761619.319074:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1053 00000100:00080000:3.0:1789761619.319080:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761619.319085:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789761619.319098:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761619.319110:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024612352:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789761619.319117:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872e680 x1876700024612480/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761619.319133:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761619.319137:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761619.319139:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761619.319143:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1053 00000100:00080000:3.0:1789761619.319148:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761619.319153:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789761619.319159:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761619.319173:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872ea00 x1876700024612608/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789761619.319248:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024612480:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789761619.319285:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024612608:192.168.203.107@tcp:38:kworker.0 00010000:00010000:0.0:1789761619.575000:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761619.830973:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761620.598987:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761620.854974:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761621.623037:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761621.878978:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000400:00000100:2.0:1789761622.391036:0:2330:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.107:988 00000400:00000100:2.0:1789761622.391083:0:2330:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.107@tcp at host 192.168.203.107 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1789761622.391099:0:2330:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.107@tcp 00000800:00000100:2.0:1789761622.391133:0:2330:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.107@tcp 00000800:00000100:2.0:1789761622.391139:0:2330:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.107@tcp 00000100:00100000:1.0:1789761622.391183:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761619/real 1789761622] req@ffffa06c1872e680 x1876700024612480/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761644 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761622.391209:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872e680 x1876700024612480/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761644 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761622.391238:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761622.391242:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761622.391247:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024612480:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789761622.391261:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761619/real 1789761622] req@ffffa06c1872ea00 x1876700024612608/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761644 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761622.391277:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872ea00 x1876700024612608/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761644 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761622.391286:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761622.391289:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761622.391293:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024612608:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789761622.391301:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761619/real 1789761622] req@ffffa06c1872f480 x1876700024612352/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761644 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761622.391314:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872f480 x1876700024612352/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761644 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761622.391321:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761622.391324:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761622.391328:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024612352:192.168.203.107@tcp:250:kworker.0 00010000:00010000:3.0:1789761622.646947:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761622.903071:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761623.671044:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761623.926971:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789761624.438998:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761624.439019:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789761624.439024:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761624.439037:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1058 00000100:00080000:2.0:1789761624.439052:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761624.439060:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789761624.439105:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50e00 x1876700024612736/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761624.439141:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761624.439146:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761624.439147:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761624.439152:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1058 00000100:00080000:2.0:1789761624.439155:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761624.439175:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789761624.439188:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761624.439204:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51500 x1876700024612864/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761624.439218:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761624.439223:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761624.439225:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761624.439231:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1058 00000100:00080000:2.0:1789761624.439236:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761624.439242:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789761624.439248:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761624.439256:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024612736:192.168.203.107@tcp:250:kworker.0 00000100:00100000:2.0:1789761624.439273:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50000 x1876700024612992/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789761624.439354:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761624.439424:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761624/real 1789761624] req@ffffa06c07e50e00 x1876700024612736/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761649 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761624.439447:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50e00 x1876700024612736/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761649 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761624.439466:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761624.439472:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761624.439477:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024612736:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789761624.439485:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024612864:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761624.439514:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761624.439540:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761624/real 1789761624] req@ffffa06c07e51500 x1876700024612864/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761649 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761624.439555:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51500 x1876700024612864/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761649 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761624.439565:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761624.439568:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761624.439572:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024612864:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789761624.439604:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024612992:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761624.439635:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761624.439649:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761624/real 1789761624] req@ffffa06c07e50000 x1876700024612992/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761649 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761624.439663:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50000 x1876700024612992/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761649 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761624.439686:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761624.439691:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761624.439695:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024612992:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789761624.695008:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761624.950942:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761625.718960:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761625.974944:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761626.742975:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761626.998905:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761627.766970:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761628.022951:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761628.791010:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761629.046925:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789761629.558922:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761629.558931:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789761629.558934:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761629.558942:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1063 00000100:00080000:2.0:1789761629.558954:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761629.558962:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789761629.558999:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e52300 x1876700024613120/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761629.559020:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761629.559023:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761629.559037:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761629.559041:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1063 00000100:00100000:1.0:1789761629.559047:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e52300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024613120:192.168.203.107@tcp:250:kworker.0 00000100:00080000:2.0:1789761629.559047:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761629.559055:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789761629.559063:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761629.559078:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51880 x1876700024613248/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761629.559091:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761629.559095:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761629.559097:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:1.0:1789761629.559103:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00080000:2.0:1789761629.559104:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1063 00000100:00080000:2.0:1789761629.559111:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761629.559117:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789761629.559123:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761629.559134:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51180 x1876700024613376/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789761629.559140:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761629/real 1789761629] req@ffffa06c07e52300 x1876700024613120/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761654 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761629.559155:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e52300 x1876700024613120/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761654 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761629.559164:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761629.559167:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761629.559170:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e52300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024613120:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789761629.559187:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024613248:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761629.559201:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761629.559208:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761629/real 1789761629] req@ffffa06c07e51880 x1876700024613248/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761654 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761629.559233:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51880 x1876700024613248/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761654 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761629.559239:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761629.559241:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761629.559244:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024613248:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789761629.559248:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024613376:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761629.559261:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761629.559270:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761629/real 1789761629] req@ffffa06c07e51180 x1876700024613376/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761654 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761629.559282:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51180 x1876700024613376/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761654 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761629.559290:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761629.559294:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761629.559297:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024613376:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789761629.814940:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761630.070902:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761630.839127:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761631.095092:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761631.862999:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761632.118971:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761632.887038:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761633.142975:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761633.910936:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761634.166939:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789761634.678984:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761634.678991:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789761634.678995:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761634.679005:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1068 00000100:00080000:2.0:1789761634.679016:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761634.679024:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789761634.679060:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51180 x1876700024613504/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761634.679092:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761634.679095:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761634.679097:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761634.679100:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1068 00000100:00080000:2.0:1789761634.679104:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761634.679110:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789761634.679121:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761634.679141:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51880 x1876700024613632/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761634.679143:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024613504:192.168.203.107@tcp:250:kworker.0 00000100:00080000:2.0:1789761634.679164:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761634.679168:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761634.679170:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761634.679174:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1068 00000100:00080000:2.0:1789761634.679179:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761634.679184:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789761634.679189:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761634.679201:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e52300 x1876700024613760/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789761634.679214:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789761634.679338:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761634/real 1789761634] req@ffffa06c07e51180 x1876700024613504/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761659 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761634.679357:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51180 x1876700024613504/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761659 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761634.679366:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761634.679368:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789761634.679371:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024613504:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789761634.679395:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024613632:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789761634.679419:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789761634.679541:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761634/real 1789761634] req@ffffa06c07e51880 x1876700024613632/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761659 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761634.679553:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51880 x1876700024613632/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761659 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761634.679561:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761634.679564:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789761634.679567:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024613632:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789761634.679572:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e52300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024613760:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789761634.679600:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789761634.679611:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761634/real 1789761634] req@ffffa06c07e52300 x1876700024613760/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761659 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761634.679623:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e52300 x1876700024613760/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761659 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761634.679630:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761634.679634:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789761634.679636:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e52300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024613760:192.168.203.107@tcp:38:kworker.0 00010000:00010000:1.0:1789761634.934986:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761635.190935:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761635.958944:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761636.214938:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761636.982961:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761637.238944:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761638.006970:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761638.262946:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761639.030977:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761639.286985:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789761639.798902:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761639.798913:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789761639.798922:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761639.798932:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1073 00000100:00080000:2.0:1789761639.798956:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761639.798963:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789761639.798994:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50000 x1876700024613888/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761639.799015:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761639.799017:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761639.799019:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761639.799022:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1073 00000100:00080000:2.0:1789761639.799027:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761639.799032:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789761639.799041:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761639.799048:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024613888:192.168.203.107@tcp:250:kworker.0 00000100:00100000:2.0:1789761639.799058:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51500 x1876700024614016/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761639.799072:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761639.799076:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761639.799079:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761639.799083:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1073 00000100:00080000:2.0:1789761639.799099:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761639.799103:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789761639.799108:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1789761639.799114:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789761639.799121:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50e00 x1876700024614144/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761639.799157:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761639/real 1789761639] req@ffffa06c07e50000 x1876700024613888/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761664 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761639.799171:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50000 x1876700024613888/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761664 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761639.799179:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761639.799182:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789761639.799185:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024613888:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789761639.799203:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024614016:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789761639.799219:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789761639.799236:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761639/real 1789761639] req@ffffa06c07e51500 x1876700024614016/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761664 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761639.799244:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51500 x1876700024614016/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761664 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761639.799250:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761639.799253:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789761639.799255:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024614016:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789761639.799259:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024614144:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789761639.799272:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789761639.799278:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761639/real 1789761639] req@ffffa06c07e50e00 x1876700024614144/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761664 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761639.799285:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50e00 x1876700024614144/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761664 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761639.799290:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761639.799292:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789761639.799294:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024614144:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789761640.054979:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761640.310923:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761641.079004:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761641.334942:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761642.102945:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761642.358957:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761643.126971:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761643.382937:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761644.150964:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761644.406925:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:3.0:1789761644.918979:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761644.918991:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789761644.919011:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761644.919019:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1078 00000100:00080000:3.0:1789761644.919028:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761644.919033:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:3.0:1789761644.919062:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872fb80 x1876700024614272/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761644.919087:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761644.919090:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761644.919091:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761644.919094:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1078 00000100:00080000:3.0:1789761644.919096:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761644.919098:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789761644.919105:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761644.919114:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872f480 x1876700024614400/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761644.919124:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761644.919126:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761644.919128:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761644.919130:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1078 00000100:00080000:3.0:1789761644.919132:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761644.919135:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789761644.919138:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761644.919146:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872ea00 x1876700024614528/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789761644.919148:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024614272:192.168.203.107@tcp:250:kworker.0 00000800:00000100:1.0:1789761644.919214:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761644.919253:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761644/real 1789761644] req@ffffa06c1872fb80 x1876700024614272/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761669 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761644.919272:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872fb80 x1876700024614272/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761669 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761644.919283:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761644.919287:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761644.919291:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024614272:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789761644.919299:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024614400:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761644.919321:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761644.919334:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761644/real 1789761644] req@ffffa06c1872f480 x1876700024614400/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761669 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761644.919347:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872f480 x1876700024614400/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761669 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761644.919356:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761644.919360:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761644.919386:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024614400:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789761644.919413:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024614528:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761644.919436:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761644.919448:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761644/real 1789761644] req@ffffa06c1872ea00 x1876700024614528/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761669 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761644.919460:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872ea00 x1876700024614528/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761669 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761644.919468:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761644.919472:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761644.919476:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024614528:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789761645.174963:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761645.430938:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761646.198979:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761646.454909:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761647.222963:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761647.478942:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761648.247003:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761648.502999:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761649.271000:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761649.527005:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789761650.039003:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761650.039014:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789761650.039019:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761650.039029:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1083 00000100:00080000:2.0:1789761650.039042:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761650.039050:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789761650.039103:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e53b80 x1876700024614656/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761650.039132:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761650.039135:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761650.039137:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761650.039140:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1083 00000100:00080000:2.0:1789761650.039144:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761650.039147:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789761650.039157:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761650.039172:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50380 x1876700024614784/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761650.039183:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761650.039186:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761650.039187:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761650.039192:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1083 00000100:00100000:1.0:1789761650.039195:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e53b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024614656:192.168.203.107@tcp:250:kworker.0 00000100:00080000:2.0:1789761650.039195:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761650.039200:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789761650.039207:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761650.039235:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51f80 x1876700024614912/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789761650.039289:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761650.039363:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761650/real 1789761650] req@ffffa06c07e53b80 x1876700024614656/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761675 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761650.039385:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e53b80 x1876700024614656/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761675 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761650.039399:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761650.039404:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761650.039417:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e53b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024614656:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789761650.039450:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024614784:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761650.039479:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761650.039495:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761650/real 1789761650] req@ffffa06c07e50380 x1876700024614784/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761675 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761650.039509:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50380 x1876700024614784/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761675 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761650.039519:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761650.039523:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761650.039528:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024614784:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789761650.039535:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024614912:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761650.039560:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761650.039573:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761650/real 1789761650] req@ffffa06c07e51f80 x1876700024614912/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761675 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761650.039586:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51f80 x1876700024614912/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761675 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761650.039595:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761650.039598:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761650.039602:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024614912:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789761650.295044:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761650.550957:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761651.319009:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761651.574955:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761652.343000:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761652.598937:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761653.366996:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761653.622936:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761654.391035:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761654.646941:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789761655.158960:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761655.158970:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761655.158975:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761655.158986:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1088 00000100:00080000:0.0:1789761655.158999:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761655.159007:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789761655.159057:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d3100 x1876700024615040/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761655.159087:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761655.159091:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761655.159092:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761655.159096:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1088 00000100:00080000:0.0:1789761655.159098:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761655.159100:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789761655.159110:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761655.159119:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d2680 x1876700024615168/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761655.159128:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761655.159131:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761655.159132:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761655.159134:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1088 00000100:00080000:0.0:1789761655.159136:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761655.159140:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789761655.159142:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024615040:192.168.203.107@tcp:250:kworker.0 00000008:00100000:0.0:1789761655.159144:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761655.159178:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0380 x1876700024615296/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789761655.159216:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761655.159256:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761655/real 1789761655] req@ffffa06c091d3100 x1876700024615040/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761680 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761655.159272:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d3100 x1876700024615040/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761680 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761655.159281:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761655.159285:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761655.159288:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024615040:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789761655.159308:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024615168:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761655.159327:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761655.159337:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761655/real 1789761655] req@ffffa06c091d2680 x1876700024615168/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761680 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761655.159346:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d2680 x1876700024615168/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761680 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761655.159352:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761655.159355:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761655.159357:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024615168:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789761655.159380:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024615296:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761655.159392:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761655.159407:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761655/real 1789761655] req@ffffa06c091d0380 x1876700024615296/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761680 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761655.159424:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0380 x1876700024615296/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761680 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761655.159430:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761655.159432:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761655.159434:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024615296:192.168.203.107@tcp:38:kworker.0 00010000:00010000:2.0:1789761655.415012:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761655.670936:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761656.439059:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761656.694993:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761657.462991:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761657.718938:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761658.487065:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761658.742998:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761659.510983:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761659.766933:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789761660.279009:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761660.279023:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789761660.279028:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761660.279043:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1094 00000100:00080000:2.0:1789761660.279058:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761660.279066:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789761660.279110:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e52680 x1876700024615424/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761660.279144:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761660.279149:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761660.279151:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761660.279157:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1094 00000100:00080000:2.0:1789761660.279162:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761660.279168:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789761660.279181:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761660.279189:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e52680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024615424:192.168.203.107@tcp:250:kworker.0 00000100:00100000:2.0:1789761660.279201:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e53800 x1876700024615552/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761660.279218:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761660.279222:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761660.279244:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761660.279261:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1094 00000100:00080000:2.0:1789761660.279266:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761660.279270:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789761660.279275:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1789761660.279289:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789761660.279290:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51c00 x1876700024615680/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789761660.279366:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761660/real 1789761660] req@ffffa06c07e52680 x1876700024615424/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761685 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761660.279390:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e52680 x1876700024615424/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761685 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761660.279403:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761660.279408:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761660.279413:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e52680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024615424:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789761660.279444:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e53800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024615552:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761660.279482:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761660.279498:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761660/real 1789761660] req@ffffa06c07e53800 x1876700024615552/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761685 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761660.279512:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e53800 x1876700024615552/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761685 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761660.279521:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761660.279525:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761660.279528:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e53800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024615552:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789761660.279535:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024615680:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761660.279557:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761660.279570:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761660/real 1789761660] req@ffffa06c07e51c00 x1876700024615680/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761685 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761660.279584:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51c00 x1876700024615680/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761685 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761660.279593:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761660.279597:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761660.279601:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024615680:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789761660.535011:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761660.790981:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761661.558950:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761661.814950:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761662.582951:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761662.838964:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761663.607000:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761663.862931:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761664.631050:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761664.886956:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:1.0:1789761665.398957:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761665.398967:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789761665.398971:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761665.398982:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1099 00000100:00080000:1.0:1789761665.398993:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761665.399000:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789761665.399038:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2d500 x1876700024615808/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761665.399067:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761665.399069:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761665.399071:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761665.399074:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1099 00000100:00080000:1.0:1789761665.399077:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761665.399081:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789761665.399088:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761665.399098:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2e300 x1876700024615936/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761665.399106:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761665.399109:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761665.399110:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761665.399113:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1099 00000100:00080000:1.0:1789761665.399116:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761665.399118:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789761665.399121:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761665.399136:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024615808:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789761665.399143:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2c000 x1876700024616064/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789761665.399192:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789761665.399243:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761665/real 1789761665] req@ffffa06c08e2d500 x1876700024615808/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761690 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761665.399257:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2d500 x1876700024615808/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761690 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761665.399266:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761665.399268:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789761665.399271:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024615808:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789761665.399276:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024615936:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789761665.399293:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789761665.399301:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761665/real 1789761665] req@ffffa06c08e2e300 x1876700024615936/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761690 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761665.399321:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2e300 x1876700024615936/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761690 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761665.399327:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761665.399329:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789761665.399331:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024615936:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789761665.399348:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024616064:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789761665.399361:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789761665.399368:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761665/real 1789761665] req@ffffa06c08e2c000 x1876700024616064/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761690 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761665.399375:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2c000 x1876700024616064/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761690 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761665.399380:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761665.399382:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789761665.399385:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024616064:192.168.203.107@tcp:38:kworker.0 00010000:00010000:2.0:1789761665.654992:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761665.910927:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761666.678984:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761666.934945:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789761667.702999:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761667.958926:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761668.727022:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761668.982949:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761669.750969:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761670.006974:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:3.0:1789761670.519032:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761670.519040:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789761670.519045:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761670.519057:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1104 00000100:00080000:3.0:1789761670.519069:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761670.519077:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:3.0:1789761670.519149:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872e680 x1876700024616192/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761670.519185:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761670.519189:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761670.519191:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761670.519196:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1104 00000100:00080000:3.0:1789761670.519203:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761670.519207:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789761670.519217:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024616192:192.168.203.107@tcp:250:kworker.0 00000008:00100000:3.0:1789761670.519219:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761670.519246:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872dc00 x1876700024616320/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761670.519265:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761670.519271:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761670.519275:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761670.519279:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1104 00000100:00080000:3.0:1789761670.519284:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761670.519288:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789761670.519293:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1789761670.519301:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789761670.519306:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872c700 x1876700024616448/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789761670.519355:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761670/real 1789761670] req@ffffa06c1872e680 x1876700024616192/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761695 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761670.519375:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872e680 x1876700024616192/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761695 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761670.519385:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761670.519391:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761670.519396:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024616192:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789761670.519423:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024616320:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761670.519452:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761670.519464:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761670/real 1789761670] req@ffffa06c1872dc00 x1876700024616320/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761695 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761670.519472:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872dc00 x1876700024616320/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761695 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761670.519478:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761670.519486:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761670.519489:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024616320:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789761670.519494:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024616448:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761670.519509:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761670.519516:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761670/real 1789761670] req@ffffa06c1872c700 x1876700024616448/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761695 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761670.519524:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872c700 x1876700024616448/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761695 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761670.519530:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761670.519531:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761670.519533:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024616448:192.168.203.107@tcp:38:kworker.0 00010000:00010000:1.0:1789761670.774988:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761671.030934:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761671.798984:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761672.054931:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761672.822959:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761673.078915:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761673.846963:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761674.103003:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761674.871024:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761675.126961:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789761675.638945:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761675.638953:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789761675.638957:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761675.638965:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1109 00000100:00080000:2.0:1789761675.638974:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761675.638980:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789761675.639010:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50a80 x1876700024616576/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761675.639030:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761675.639033:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761675.639034:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761675.639037:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1109 00000100:00080000:2.0:1789761675.639051:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761675.639054:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789761675.639062:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761675.639071:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50700 x1876700024616704/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789761675.639088:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024616576:192.168.203.107@tcp:250:kworker.0 00000100:00080000:2.0:1789761675.639090:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761675.639098:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761675.639099:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761675.639102:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1109 00000100:00080000:2.0:1789761675.639105:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761675.639107:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789761675.639110:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761675.639119:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e53480 x1876700024616832/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789761675.639171:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761675.639210:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761675/real 1789761675] req@ffffa06c07e50a80 x1876700024616576/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761700 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761675.639228:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50a80 x1876700024616576/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761700 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761675.639247:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761675.639251:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761675.639255:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024616576:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789761675.639279:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024616704:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761675.639305:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761675.639323:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761675/real 1789761675] req@ffffa06c07e50700 x1876700024616704/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761700 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761675.639336:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50700 x1876700024616704/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761700 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761675.639345:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761675.639349:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761675.639352:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024616704:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789761675.639358:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e53480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024616832:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761675.639376:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761675.639387:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761675/real 1789761675] req@ffffa06c07e53480 x1876700024616832/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761700 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761675.639398:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e53480 x1876700024616832/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761700 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761675.639406:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761675.639409:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761675.639413:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e53480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024616832:192.168.203.107@tcp:38:kworker.0 00010000:00010000:2.0:1789761675.895019:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761676.150986:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761676.919060:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761677.174979:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761677.943045:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761678.199002:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761678.966992:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761679.222940:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761679.991003:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761680.246947:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789761680.758964:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761680.758975:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761680.758979:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761680.758989:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1114 00000100:00080000:0.0:1789761680.759001:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761680.759009:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789761680.759044:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d1f80 x1876700024616960/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761680.759065:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761680.759068:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761680.759070:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761680.759073:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1114 00000100:00080000:0.0:1789761680.759076:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761680.759078:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789761680.759086:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761680.759095:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0a80 x1876700024617088/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789761680.759118:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024616960:192.168.203.107@tcp:250:kworker.0 00000100:00080000:0.0:1789761680.759121:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761680.759128:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761680.759129:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761680.759133:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1114 00000100:00080000:0.0:1789761680.759148:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761680.759152:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789761680.759157:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761680.759166:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0700 x1876700024617216/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789761680.759178:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761680.759212:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761680/real 1789761680] req@ffffa06c091d1f80 x1876700024616960/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761705 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761680.759226:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d1f80 x1876700024616960/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761705 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761680.759234:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761680.759236:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761680.759239:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024616960:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789761680.759263:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024617088:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761680.759277:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761680.759286:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761680/real 1789761680] req@ffffa06c091d0a80 x1876700024617088/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761705 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761680.759294:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0a80 x1876700024617088/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761705 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761680.759300:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761680.759302:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761680.759310:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024617088:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789761680.759315:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024617216:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761680.759329:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761680.759335:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761680/real 1789761680] req@ffffa06c091d0700 x1876700024617216/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761705 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761680.759343:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0700 x1876700024617216/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761705 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761680.759349:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761680.759350:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761680.759357:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024617216:192.168.203.107@tcp:38:kworker.0 00010000:00010000:2.0:1789761681.014972:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761681.270941:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789761682.039009:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761682.294962:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761683.062942:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761683.318909:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761684.087039:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761684.342964:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761685.110954:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761685.366922:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:1.0:1789761685.878920:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761685.878928:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789761685.878932:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761685.878940:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1119 00000100:00080000:1.0:1789761685.878949:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761685.878954:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789761685.878983:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2c700 x1876700024617344/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761685.879006:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761685.879018:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761685.879019:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761685.879023:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1119 00000100:00080000:1.0:1789761685.879027:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761685.879030:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789761685.879038:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761685.879048:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024617344:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789761685.879050:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2ca80 x1876700024617472/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761685.879076:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761685.879080:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761685.879081:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761685.879084:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1119 00000100:00080000:1.0:1789761685.879087:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761685.879090:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789761685.879094:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761685.879103:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2ed80 x1876700024617600/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761685.879173:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024617472:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789761685.879199:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024617600:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789761686.134983:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761686.390940:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761687.158971:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761687.414910:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761688.182947:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761688.438994:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000400:00000100:1.0:1789761688.951058:0:2331:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.107:988 00000400:00000100:1.0:1789761688.951100:0:2331:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.107@tcp at host 192.168.203.107 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1789761688.951114:0:2331:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.107@tcp 00000800:00000100:1.0:1789761688.951150:0:2331:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.107@tcp 00000800:00000100:1.0:1789761688.951176:0:2331:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.107@tcp 00000100:00100000:3.0:1789761688.951219:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761685/real 1789761688] req@ffffa06c08e2ca80 x1876700024617472/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761710 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761688.951257:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2ca80 x1876700024617472/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761710 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761688.951272:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761688.951278:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789761688.951284:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024617472:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789761688.951294:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761685/real 1789761688] req@ffffa06c08e2ed80 x1876700024617600/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761710 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761688.951310:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2ed80 x1876700024617600/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761710 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761688.951319:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761688.951324:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789761688.951328:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024617600:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789761688.951336:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761685/real 1789761688] req@ffffa06c08e2c700 x1876700024617344/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761710 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761688.951351:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2c700 x1876700024617344/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761710 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761688.951377:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761688.951381:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789761688.951385:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024617344:192.168.203.107@tcp:250:kworker.0 00010000:00010000:3.0:1789761689.207013:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761689.462995:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761690.230989:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761690.486972:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789761690.998980:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761690.998995:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789761690.999000:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761690.999013:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1124 00000100:00080000:2.0:1789761690.999039:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761690.999060:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789761690.999098:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e52a00 x1876700024617728/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761690.999123:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761690.999127:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761690.999128:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761690.999132:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1124 00000100:00080000:2.0:1789761690.999135:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761690.999138:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789761690.999147:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761690.999159:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e53100 x1876700024617856/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761690.999174:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761690.999178:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761690.999181:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1789761690.999186:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e52a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024617728:192.168.203.107@tcp:250:kworker.0 00000100:00080000:2.0:1789761690.999188:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1124 00000100:00080000:2.0:1789761690.999195:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761690.999201:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789761690.999207:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761690.999221:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e52d80 x1876700024617984/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789761690.999280:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789761690.999339:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761690/real 1789761690] req@ffffa06c07e52a00 x1876700024617728/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761715 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761690.999363:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e52a00 x1876700024617728/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761715 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761690.999375:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761690.999380:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789761690.999384:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e52a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024617728:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789761690.999416:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e53100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024617856:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789761690.999443:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789761690.999458:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761690/real 1789761690] req@ffffa06c07e53100 x1876700024617856/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761715 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761690.999473:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e53100 x1876700024617856/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761715 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761690.999482:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761690.999486:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789761690.999501:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e53100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024617856:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789761690.999509:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e52d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024617984:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789761690.999534:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789761690.999548:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761690/real 1789761690] req@ffffa06c07e52d80 x1876700024617984/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761715 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761690.999562:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e52d80 x1876700024617984/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761715 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761690.999570:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761690.999573:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789761690.999576:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e52d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024617984:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789761691.255000:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761691.510966:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761692.278975:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761692.534949:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761693.303001:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761693.558937:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761694.327028:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761694.582929:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761695.350985:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761695.606912:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789761696.118953:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761696.118967:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789761696.119011:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761696.119023:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1129 00000100:00080000:2.0:1789761696.119035:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761696.119042:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789761696.119090:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e52d80 x1876700024618112/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761696.119117:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761696.119121:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761696.119122:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761696.119126:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1129 00000100:00080000:2.0:1789761696.119128:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761696.119131:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789761696.119140:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761696.119150:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e53100 x1876700024618240/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761696.119160:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761696.119165:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:3.0:1789761696.119166:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e52d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024618112:192.168.203.107@tcp:250:kworker.0 00000100:00080000:2.0:1789761696.119167:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761696.119175:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1129 00000100:00080000:2.0:1789761696.119178:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761696.119180:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789761696.119184:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761696.119193:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e52a00 x1876700024618368/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789761696.119244:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789761696.119291:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761696/real 1789761696] req@ffffa06c07e52d80 x1876700024618112/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761721 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761696.119312:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e52d80 x1876700024618112/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761721 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761696.119326:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761696.119331:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789761696.119336:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e52d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024618112:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789761696.119376:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e53100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024618240:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789761696.119409:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789761696.119423:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761696/real 1789761696] req@ffffa06c07e53100 x1876700024618240/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761721 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761696.119435:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e53100 x1876700024618240/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761721 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761696.119444:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761696.119448:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789761696.119452:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e53100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024618240:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789761696.119459:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e52a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024618368:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789761696.119489:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789761696.119500:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761696/real 1789761696] req@ffffa06c07e52a00 x1876700024618368/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761721 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761696.119508:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e52a00 x1876700024618368/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761721 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761696.119514:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761696.119517:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789761696.119519:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e52a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024618368:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789761696.374975:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761696.630933:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761697.398947:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761697.654945:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761698.422949:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761698.678914:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761699.446954:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761699.702973:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761700.470934:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761700.726922:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789761701.238969:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761701.238980:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761701.238985:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761701.238994:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1135 00000100:00080000:0.0:1789761701.239006:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761701.239014:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789761701.239053:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0700 x1876700024618496/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761701.239086:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761701.239091:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761701.239093:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761701.239098:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1135 00000100:00080000:0.0:1789761701.239103:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761701.239108:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789761701.239118:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761701.239126:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024618496:192.168.203.107@tcp:250:kworker.0 00000100:00100000:0.0:1789761701.239135:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0a80 x1876700024618624/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761701.239156:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761701.239169:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761701.239171:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761701.239177:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1135 00000100:00080000:0.0:1789761701.239182:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761701.239188:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789761701.239194:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1789761701.239198:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789761701.239214:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d1f80 x1876700024618752/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761701.239236:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761701/real 1789761701] req@ffffa06c091d0700 x1876700024618496/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761726 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761701.239252:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0700 x1876700024618496/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761726 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761701.239260:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761701.239263:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789761701.239265:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024618496:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789761701.239282:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024618624:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789761701.239298:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789761701.239306:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761701/real 1789761701] req@ffffa06c091d0a80 x1876700024618624/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761726 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761701.239318:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0a80 x1876700024618624/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761726 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761701.239326:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761701.239330:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789761701.239333:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024618624:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789761701.239340:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024618752:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789761701.239362:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789761701.239374:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761701/real 1789761701] req@ffffa06c091d1f80 x1876700024618752/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761726 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761701.239394:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d1f80 x1876700024618752/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761726 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761701.239400:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761701.239402:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789761701.239404:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024618752:192.168.203.107@tcp:38:kworker.0 00010000:00010000:1.0:1789761701.495000:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761701.750975:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761702.518993:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761702.774890:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761703.542973:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761703.798958:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761704.566968:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761704.822942:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761705.590969:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761705.846950:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789761706.358934:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761706.358944:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761706.358948:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761706.358957:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1140 00000100:00080000:0.0:1789761706.358967:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761706.358973:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789761706.359001:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0380 x1876700024618880/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761706.359024:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761706.359027:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761706.359029:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761706.359032:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1140 00000100:00080000:0.0:1789761706.359034:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761706.359037:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789761706.359044:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761706.359055:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d2680 x1876700024619008/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789761706.359067:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024618880:192.168.203.107@tcp:250:kworker.0 00000100:00080000:0.0:1789761706.359076:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761706.359080:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761706.359082:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761706.359087:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1140 00000100:00080000:0.0:1789761706.359091:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761706.359094:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789761706.359100:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761706.359116:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d3100 x1876700024619136/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789761706.359134:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761706.359187:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761706/real 1789761706] req@ffffa06c091d0380 x1876700024618880/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761731 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761706.359209:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0380 x1876700024618880/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761731 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761706.359219:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761706.359231:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761706.359236:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024618880:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789761706.359260:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024619008:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761706.359287:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761706.359299:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761706/real 1789761706] req@ffffa06c091d2680 x1876700024619008/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761731 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761706.359308:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d2680 x1876700024619008/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761731 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761706.359314:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761706.359317:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761706.359319:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024619008:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789761706.359324:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024619136:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761706.359338:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761706.359346:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761706/real 1789761706] req@ffffa06c091d3100 x1876700024619136/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761731 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761706.359354:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d3100 x1876700024619136/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761731 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761706.359360:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761706.359362:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761706.359366:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024619136:192.168.203.107@tcp:38:kworker.0 00010000:00010000:2.0:1789761706.614969:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761706.870940:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761707.638955:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761707.894943:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761708.662970:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761708.918974:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761709.686986:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761709.943002:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761710.711080:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761710.966995:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789761711.478966:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761711.478977:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761711.478981:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761711.478992:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1145 00000100:00080000:0.0:1789761711.479003:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761711.479011:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789761711.479049:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d1500 x1876700024619264/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761711.479077:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761711.479081:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761711.479083:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761711.479088:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1145 00000100:00080000:0.0:1789761711.479092:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761711.479098:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789761711.479119:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024619264:192.168.203.107@tcp:250:kworker.0 00000008:00100000:0.0:1789761711.479124:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761711.479143:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d2a00 x1876700024619392/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761711.479157:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761711.479161:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761711.479163:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761711.479167:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1145 00000100:00080000:0.0:1789761711.479172:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761711.479178:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000800:00000100:1.0:1789761711.479181:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000008:00100000:0.0:1789761711.479187:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761711.479201:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0e00 x1876700024619520/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789761711.479219:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761711/real 1789761711] req@ffffa06c091d1500 x1876700024619264/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761736 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761711.479234:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d1500 x1876700024619264/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761736 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761711.479243:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761711.479246:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761711.479248:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024619264:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789761711.479266:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024619392:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761711.479281:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761711.479289:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761711/real 1789761711] req@ffffa06c091d2a00 x1876700024619392/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761736 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761711.479297:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d2a00 x1876700024619392/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761736 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761711.479302:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761711.479304:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761711.479306:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024619392:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789761711.479317:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024619520:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761711.479330:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761711.479337:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761711/real 1789761711] req@ffffa06c091d0e00 x1876700024619520/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761736 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761711.479345:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0e00 x1876700024619520/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761736 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761711.479350:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761711.479357:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761711.479359:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024619520:192.168.203.107@tcp:38:kworker.0 00010000:00010000:0.0:1789761711.735047:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761711.990961:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761712.758969:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761713.014992:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789761713.782958:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761714.038951:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761714.807059:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761715.062927:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761715.830930:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761716.086948:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:1.0:1789761716.598980:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761716.598991:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789761716.598996:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761716.599007:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1150 00000100:00080000:1.0:1789761716.599027:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761716.599036:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789761716.599075:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2d180 x1876700024619648/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761716.599117:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761716.599121:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761716.599139:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761716.599144:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1150 00000100:00080000:1.0:1789761716.599150:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761716.599155:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789761716.599166:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761716.599168:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024619648:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789761716.599185:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2c700 x1876700024619776/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761716.599200:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761716.599204:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761716.599206:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761716.599210:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1150 00000100:00080000:1.0:1789761716.599214:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761716.599220:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789761716.599226:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1789761716.599236:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761716.599242:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2ed80 x1876700024619904/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761716.599284:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761716/real 1789761716] req@ffffa06c08e2d180 x1876700024619648/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761741 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761716.599305:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2d180 x1876700024619648/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761741 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761716.599315:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761716.599319:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789761716.599323:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024619648:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789761716.599346:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024619776:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789761716.599371:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789761716.599386:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761716/real 1789761716] req@ffffa06c08e2c700 x1876700024619776/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761741 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761716.599399:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2c700 x1876700024619776/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761741 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761716.599416:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761716.599420:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789761716.599424:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024619776:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789761716.599430:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024619904:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789761716.599454:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789761716.599467:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761716/real 1789761716] req@ffffa06c08e2ed80 x1876700024619904/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761741 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761716.599479:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2ed80 x1876700024619904/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761741 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761716.599487:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761716.599491:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789761716.599494:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024619904:192.168.203.107@tcp:38:kworker.0 00010000:00010000:2.0:1789761716.855000:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761717.110929:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761717.878973:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761718.134934:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761718.902973:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761719.158972:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761719.926958:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761720.182955:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761720.950969:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761721.206962:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789761721.718966:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761721.718974:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789761721.718977:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761721.718985:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1155 00000100:00080000:2.0:1789761721.718995:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761721.719000:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789761721.719035:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e53480 x1876700024620032/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761721.719064:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761721.719069:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761721.719071:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761721.719075:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1155 00000100:00080000:2.0:1789761721.719080:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761721.719084:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789761721.719094:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761721.719109:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50700 x1876700024620160/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761721.719128:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e53480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024620032:192.168.203.107@tcp:250:kworker.0 00000100:00080000:2.0:1789761721.719139:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761721.719142:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761721.719144:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761721.719148:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1155 00000100:00080000:2.0:1789761721.719152:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761721.719154:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789761721.719159:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761721.719172:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50a80 x1876700024620288/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789761721.719194:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789761721.719230:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761721/real 1789761721] req@ffffa06c07e53480 x1876700024620032/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761746 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761721.719243:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e53480 x1876700024620032/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761746 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761721.719261:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761721.719264:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789761721.719267:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e53480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024620032:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789761721.719289:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024620160:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789761721.719316:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789761721.719331:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761721/real 1789761721] req@ffffa06c07e50700 x1876700024620160/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761746 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761721.719342:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50700 x1876700024620160/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761746 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761721.719348:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761721.719350:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789761721.719352:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024620160:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789761721.719357:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024620288:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789761721.719372:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789761721.719379:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761721/real 1789761721] req@ffffa06c07e50a80 x1876700024620288/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761746 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761721.719387:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50a80 x1876700024620288/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761746 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761721.719392:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761721.719394:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789761721.719395:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024620288:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789761721.974946:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761722.230942:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761722.998957:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761723.254981:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761724.022961:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761724.278943:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761725.047021:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761725.302988:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761726.070955:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761726.326937:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:1.0:1789761726.838939:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761726.838947:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789761726.838950:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761726.838958:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1160 00000100:00080000:1.0:1789761726.838967:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761726.838973:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789761726.839013:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2ca80 x1876700024620416/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761726.839041:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761726.839045:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761726.839047:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761726.839052:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1160 00000100:00080000:1.0:1789761726.839057:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761726.839062:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789761726.839072:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761726.839083:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024620416:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789761726.839090:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2c000 x1876700024620544/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761726.839104:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761726.839107:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761726.839108:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761726.839111:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1160 00000100:00080000:1.0:1789761726.839115:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761726.839118:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789761726.839122:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1789761726.839142:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761726.839158:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2e300 x1876700024620672/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761726.839178:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761726/real 1789761726] req@ffffa06c08e2ca80 x1876700024620416/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761751 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761726.839194:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2ca80 x1876700024620416/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761751 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761726.839202:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761726.839205:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789761726.839208:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024620416:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789761726.839226:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024620544:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789761726.839241:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789761726.839248:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761726/real 1789761726] req@ffffa06c08e2c000 x1876700024620544/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761751 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761726.839257:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2c000 x1876700024620544/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761751 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761726.839262:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761726.839264:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789761726.839266:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024620544:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789761726.839279:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024620672:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789761726.839293:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789761726.839300:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761726/real 1789761726] req@ffffa06c08e2e300 x1876700024620672/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761751 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761726.839316:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2e300 x1876700024620672/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761751 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761726.839322:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761726.839324:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789761726.839326:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024620672:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789761727.094980:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761727.350970:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761728.118989:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761728.374936:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761729.142956:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761729.398930:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761730.166969:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761730.422977:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761731.190955:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761731.446998:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:1.0:1789761731.958962:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761731.958973:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789761731.958977:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761731.958987:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1165 00000100:00080000:1.0:1789761731.958997:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761731.959004:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789761731.959041:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2d500 x1876700024620800/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761731.959070:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761731.959079:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761731.959081:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761731.959085:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1165 00000100:00080000:1.0:1789761731.959090:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761731.959095:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789761731.959104:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761731.959114:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024620800:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789761731.959124:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2e300 x1876700024620928/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761731.959138:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761731.959155:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761731.959157:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761731.959162:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1165 00000100:00080000:1.0:1789761731.959168:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1789761731.959173:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00080000:1.0:1789761731.959178:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789761731.959186:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761731.959201:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2c000 x1876700024621056/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761731.959222:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761731/real 1789761731] req@ffffa06c08e2d500 x1876700024620800/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761756 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761731.959244:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2d500 x1876700024620800/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761756 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761731.959256:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761731.959260:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789761731.959264:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024620800:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789761731.959284:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024620928:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789761731.959310:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789761731.959319:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761731/real 1789761731] req@ffffa06c08e2e300 x1876700024620928/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761756 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761731.959328:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2e300 x1876700024620928/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761756 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761731.959334:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761731.959336:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789761731.959338:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024620928:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789761731.959342:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024621056:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789761731.959352:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789761731.959359:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761731/real 1789761731] req@ffffa06c08e2c000 x1876700024621056/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761756 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761731.959366:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2c000 x1876700024621056/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761756 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761731.959371:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761731.959373:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789761731.959375:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024621056:192.168.203.107@tcp:38:kworker.0 00010000:00010000:2.0:1789761732.214968:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761732.470943:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761733.238989:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761733.494953:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761734.263006:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761734.518984:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789761735.286970:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761735.542924:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761736.311055:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761736.566920:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789761737.078927:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761737.078934:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761737.078937:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761737.078944:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1170 00000100:00080000:0.0:1789761737.078953:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761737.078963:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789761737.078989:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d3b80 x1876700024621184/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761737.079009:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761737.079024:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761737.079025:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761737.079028:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1170 00000100:00080000:0.0:1789761737.079032:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761737.079036:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789761737.079045:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761737.079056:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d2300 x1876700024621312/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761737.079066:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024621184:192.168.203.107@tcp:250:kworker.0 00000100:00080000:0.0:1789761737.079070:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761737.079079:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761737.079081:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761737.079085:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1170 00000100:00080000:0.0:1789761737.079087:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761737.079090:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789761737.079093:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761737.079103:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0000 x1876700024621440/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789761737.079157:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789761737.079204:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761737/real 1789761737] req@ffffa06c091d3b80 x1876700024621184/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761762 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761737.079237:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d3b80 x1876700024621184/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761762 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761737.079246:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761737.079250:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789761737.079254:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024621184:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789761737.079280:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024621312:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789761737.079299:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789761737.079308:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761737/real 1789761737] req@ffffa06c091d2300 x1876700024621312/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761762 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761737.079316:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d2300 x1876700024621312/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761762 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761737.079323:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761737.079325:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789761737.079327:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024621312:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789761737.079331:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024621440:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789761737.079342:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789761737.079348:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761737/real 1789761737] req@ffffa06c091d0000 x1876700024621440/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761762 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761737.079356:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0000 x1876700024621440/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761762 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761737.079373:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761737.079375:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789761737.079378:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024621440:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789761737.334998:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761737.590969:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789761738.359016:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761738.614920:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761739.382964:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761739.638931:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789761740.406956:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761740.662936:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789761741.430931:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761741.686930:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789761742.198940:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761742.198948:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761742.198951:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761742.198960:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1176 00000100:00080000:0.0:1789761742.198986:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761742.198994:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789761742.199030:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0000 x1876700024621568/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761742.199059:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761742.199063:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761742.199065:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761742.199069:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1176 00000100:00080000:0.0:1789761742.199074:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761742.199079:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789761742.199089:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761742.199093:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024621568:192.168.203.107@tcp:250:kworker.0 00000100:00100000:0.0:1789761742.199108:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d2300 x1876700024621696/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761742.199122:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761742.199126:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761742.199128:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761742.199132:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1176 00000100:00080000:0.0:1789761742.199137:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761742.199142:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789761742.199147:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1789761742.199150:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789761742.199165:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d3b80 x1876700024621824/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789761742.199192:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761742/real 1789761742] req@ffffa06c091d0000 x1876700024621568/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761767 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761742.199213:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0000 x1876700024621568/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761767 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761742.199234:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761742.199240:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789761742.199244:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024621568:192.168.203.107@tcp:250:kworker.0 00000100:00100000:2.0:1789761742.199268:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024621696:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789761742.199300:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789761742.199313:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761742/real 1789761742] req@ffffa06c091d2300 x1876700024621696/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761767 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761742.199327:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d2300 x1876700024621696/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761767 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761742.199336:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761742.199340:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789761742.199343:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024621696:192.168.203.107@tcp:38:kworker.0 00000100:00100000:2.0:1789761742.199350:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024621824:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789761742.199381:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789761742.199393:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761742/real 1789761742] req@ffffa06c091d3b80 x1876700024621824/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761767 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761742.199406:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d3b80 x1876700024621824/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761767 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761742.199414:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761742.199418:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789761742.199421:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024621824:192.168.203.107@tcp:38:kworker.0 00010000:00010000:2.0:1789761742.454986:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761742.710957:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761743.478974:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761743.734907:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789761744.502956:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761744.758925:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761745.526966:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761745.782915:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761746.550965:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761746.806941:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789761747.318993:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761747.319004:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789761747.319026:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761747.319036:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1181 00000100:00080000:2.0:1789761747.319047:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761747.319054:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789761747.319089:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51c00 x1876700024621952/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761747.319115:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761747.319118:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761747.319119:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761747.319122:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1181 00000100:00080000:2.0:1789761747.319125:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761747.319128:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789761747.319135:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761747.319144:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e53800 x1876700024622080/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761747.319153:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1789761747.319154:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024621952:192.168.203.107@tcp:250:kworker.0 00000100:00080000:2.0:1789761747.319159:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761747.319162:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761747.319165:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1181 00000100:00080000:2.0:1789761747.319167:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761747.319169:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789761747.319172:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761747.319180:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e52680 x1876700024622208/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789761747.319212:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789761747.319245:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761747/real 1789761747] req@ffffa06c07e51c00 x1876700024621952/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761772 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761747.319258:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51c00 x1876700024621952/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761772 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761747.319265:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761747.319268:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789761747.319271:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024621952:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789761747.319288:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e53800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024622080:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789761747.319301:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789761747.319309:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761747/real 1789761747] req@ffffa06c07e53800 x1876700024622080/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761772 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761747.319316:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e53800 x1876700024622080/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761772 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761747.319322:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761747.319324:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789761747.319333:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e53800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024622080:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789761747.319337:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e52680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024622208:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789761747.319351:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789761747.319358:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761747/real 1789761747] req@ffffa06c07e52680 x1876700024622208/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761772 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761747.319365:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e52680 x1876700024622208/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761772 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761747.319370:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761747.319372:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789761747.319374:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e52680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024622208:192.168.203.107@tcp:38:kworker.0 00010000:00010000:1.0:1789761747.575027:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761747.830925:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761748.599076:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761748.854958:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761749.622967:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761749.879020:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761750.646988:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761750.902992:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761751.671031:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761751.927017:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:1.0:1789761752.438956:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761752.438970:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789761752.438975:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761752.438989:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1186 00000100:00080000:1.0:1789761752.439002:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761752.439012:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789761752.439062:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2ca80 x1876700024622336/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761752.439091:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761752.439094:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761752.439096:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761752.439100:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1186 00000100:00080000:1.0:1789761752.439104:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761752.439108:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789761752.439118:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761752.439126:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024622336:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789761752.439133:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2ed80 x1876700024622464/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761752.439144:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761752.439147:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761752.439149:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761752.439152:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1186 00000100:00080000:1.0:1789761752.439156:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761752.439177:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789761752.439183:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761752.439198:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2c700 x1876700024622592/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761752.439246:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024622464:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789761752.439274:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024622592:192.168.203.107@tcp:38:kworker.0 00010000:00010000:1.0:1789761752.694967:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761752.951009:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761753.719004:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761753.974996:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761754.742951:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761754.998984:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000400:00000100:2.0:1789761755.511036:0:2328:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.107:988 00000400:00000100:2.0:1789761755.511072:0:2328:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.107@tcp at host 192.168.203.107 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1789761755.511090:0:2328:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.107@tcp 00000800:00000100:2.0:1789761755.511127:0:2328:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.107@tcp 00000800:00000100:2.0:1789761755.511139:0:2328:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.107@tcp 00000100:00100000:3.0:1789761755.511162:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761752/real 1789761755] req@ffffa06c08e2ed80 x1876700024622464/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761777 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761755.511189:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2ed80 x1876700024622464/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761777 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761755.511221:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761755.511227:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789761755.511232:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024622464:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789761755.511241:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761752/real 1789761755] req@ffffa06c08e2c700 x1876700024622592/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761777 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761755.511256:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2c700 x1876700024622592/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761777 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761755.511265:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761755.511269:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789761755.511273:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024622592:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789761755.511280:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761752/real 1789761755] req@ffffa06c08e2ca80 x1876700024622336/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761777 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761755.511294:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2ca80 x1876700024622336/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761777 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761755.511309:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761755.511313:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789761755.511316:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024622336:192.168.203.107@tcp:250:kworker.0 00010000:00010000:1.0:1789761755.767008:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761756.022957:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789761756.791060:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761757.046932:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789761757.558980:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761757.558990:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789761757.558994:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761757.559003:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1191 00000100:00080000:2.0:1789761757.559011:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761757.559016:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789761757.559048:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51f80 x1876700024622720/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761757.559077:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761757.559082:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761757.559084:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761757.559089:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1191 00000100:00080000:2.0:1789761757.559093:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761757.559097:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789761757.559106:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761757.559119:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50380 x1876700024622848/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761757.559162:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761757.559166:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761757.559168:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761757.559172:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1191 00000100:00080000:2.0:1789761757.559177:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761757.559181:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789761757.559187:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761757.559200:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e53b80 x1876700024622976/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761757.559547:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024622720:192.168.203.107@tcp:250:kworker.0 00000800:00000100:3.0:1789761757.559612:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789761757.559654:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761757/real 1789761757] req@ffffa06c07e51f80 x1876700024622720/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761782 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761757.559943:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51f80 x1876700024622720/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761782 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761757.559956:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761757.559962:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789761757.559968:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024622720:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789761757.559982:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024622848:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789761757.560017:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789761757.560031:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761757/real 1789761757] req@ffffa06c07e50380 x1876700024622848/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761782 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761757.560044:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50380 x1876700024622848/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761782 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761757.560053:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761757.560056:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789761757.560060:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024622848:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789761757.560067:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e53b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024622976:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789761757.560429:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789761757.560455:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761757/real 1789761757] req@ffffa06c07e53b80 x1876700024622976/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761782 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761757.560470:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e53b80 x1876700024622976/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761782 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761757.560493:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761757.560498:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789761757.560502:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e53b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024622976:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789761757.814975:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761758.070930:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761758.838941:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761759.094933:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789761759.862988:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761760.118934:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761760.886927:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761761.142931:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761761.911003:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761762.166968:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789761762.678911:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761762.678919:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789761762.678922:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761762.678931:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1196 00000100:00080000:2.0:1789761762.678939:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761762.678945:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789761762.678977:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50e00 x1876700024623104/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761762.679000:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761762.679002:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761762.679004:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761762.679006:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1196 00000100:00080000:2.0:1789761762.679009:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761762.679011:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789761762.679033:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761762.679052:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51500 x1876700024623232/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761762.679067:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761762.679071:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761762.679072:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761762.679079:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1196 00000100:00100000:3.0:1789761762.679081:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024623104:192.168.203.107@tcp:250:kworker.0 00000100:00080000:2.0:1789761762.679083:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761762.679094:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789761762.679100:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761762.679113:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50000 x1876700024623360/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789761762.679170:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789761762.679217:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761762/real 1789761762] req@ffffa06c07e50e00 x1876700024623104/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761787 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761762.679233:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50e00 x1876700024623104/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761787 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761762.679244:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761762.679249:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789761762.679254:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024623104:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789761762.679286:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024623232:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789761762.679308:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789761762.679319:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761762/real 1789761762] req@ffffa06c07e51500 x1876700024623232/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761787 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761762.679328:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51500 x1876700024623232/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761787 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761762.679334:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761762.679349:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789761762.679352:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024623232:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789761762.679357:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024623360:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789761762.679373:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789761762.679380:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761762/real 1789761762] req@ffffa06c07e50000 x1876700024623360/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761787 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761762.679388:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50000 x1876700024623360/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761787 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761762.679394:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761762.679396:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789761762.679398:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024623360:192.168.203.107@tcp:38:kworker.0 00010000:00010000:0.0:1789761762.934976:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761763.190911:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761763.959017:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761764.214936:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761764.983032:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761765.238921:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789761766.007090:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761766.262980:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761767.030982:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761767.286965:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789761767.799030:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761767.799044:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761767.799050:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761767.799062:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1201 00000100:00080000:0.0:1789761767.799077:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761767.799087:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789761767.799132:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0e00 x1876700024623488/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761767.799160:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761767.799163:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761767.799164:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761767.799168:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1201 00000100:00080000:0.0:1789761767.799171:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761767.799174:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789761767.799185:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761767.799195:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d2a00 x1876700024623616/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761767.799205:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024623488:192.168.203.107@tcp:250:kworker.0 00000100:00080000:0.0:1789761767.799208:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761767.799213:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761767.799215:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761767.799218:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1201 00000100:00080000:0.0:1789761767.799239:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761767.799243:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789761767.799248:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761767.799258:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d1500 x1876700024623744/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789761767.799293:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789761767.799345:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761767/real 1789761767] req@ffffa06c091d0e00 x1876700024623488/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761792 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761767.799362:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0e00 x1876700024623488/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761792 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761767.799372:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761767.799376:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789761767.799379:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024623488:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789761767.799408:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024623616:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789761767.799445:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789761767.799480:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761767/real 1789761767] req@ffffa06c091d2a00 x1876700024623616/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761792 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761767.799496:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d2a00 x1876700024623616/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761792 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761767.799506:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761767.799510:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789761767.799513:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024623616:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789761767.799520:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024623744:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789761767.799546:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789761767.799559:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761767/real 1789761767] req@ffffa06c091d1500 x1876700024623744/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761792 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761767.799572:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d1500 x1876700024623744/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761792 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761767.799578:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761767.799580:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789761767.799582:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024623744:192.168.203.107@tcp:38:kworker.0 00010000:00010000:1.0:1789761768.055014:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761768.310963:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761769.078999:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761769.334982:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761770.103030:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761770.359015:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761771.126992:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761771.382966:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789761772.151011:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761772.406929:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789761772.918912:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761772.918920:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761772.918924:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761772.918933:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1206 00000100:00080000:0.0:1789761772.918941:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761772.918947:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789761772.918978:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d3100 x1876700024623872/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761772.918999:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761772.919006:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761772.919007:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761772.919010:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1206 00000100:00080000:0.0:1789761772.919013:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761772.919016:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789761772.919024:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761772.919034:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d2680 x1876700024624000/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761772.919042:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024623872:192.168.203.107@tcp:250:kworker.0 00000100:00080000:0.0:1789761772.919047:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761772.919062:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761772.919063:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761772.919067:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1206 00000100:00080000:0.0:1789761772.919071:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761772.919074:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789761772.919079:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761772.919088:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0380 x1876700024624128/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789761772.919097:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789761772.919132:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761772/real 1789761772] req@ffffa06c091d3100 x1876700024623872/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761797 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761772.919156:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d3100 x1876700024623872/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761797 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761772.919164:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761772.919168:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789761772.919172:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024623872:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789761772.919195:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024624000:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789761772.919221:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789761772.919235:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761772/real 1789761772] req@ffffa06c091d2680 x1876700024624000/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761797 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761772.919244:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d2680 x1876700024624000/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761797 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761772.919251:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761772.919253:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789761772.919256:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024624000:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789761772.919260:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024624128:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789761772.919274:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789761772.919281:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761772/real 1789761772] req@ffffa06c091d0380 x1876700024624128/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761797 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761772.919289:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0380 x1876700024624128/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761797 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761772.919303:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761772.919306:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789761772.919308:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024624128:192.168.203.107@tcp:38:kworker.0 00010000:00010000:1.0:1789761773.174995:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761773.430959:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761774.198928:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761774.454963:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761775.223090:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761775.478999:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761776.247014:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761776.502968:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761777.271014:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761777.526974:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:3.0:1789761778.038980:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761778.038990:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789761778.038995:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761778.039006:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1211 00000100:00080000:3.0:1789761778.039018:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761778.039026:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:3.0:1789761778.039064:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872ed80 x1876700024624256/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761778.039089:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761778.039092:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761778.039094:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761778.039096:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1211 00000100:00080000:3.0:1789761778.039099:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761778.039101:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789761778.039110:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761778.039119:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872f100 x1876700024624384/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761778.039134:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761778.039153:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761778.039156:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761778.039160:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1211 00000100:00080000:3.0:1789761778.039165:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761778.039171:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789761778.039177:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761778.039193:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872ca80 x1876700024624512/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789761778.039199:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024624256:192.168.203.107@tcp:250:kworker.0 00000800:00000100:2.0:1789761778.039279:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789761778.039324:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761778/real 1789761778] req@ffffa06c1872ed80 x1876700024624256/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761803 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761778.039340:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872ed80 x1876700024624256/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761803 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761778.039349:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761778.039351:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789761778.039355:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024624256:192.168.203.107@tcp:250:kworker.0 00000100:00100000:2.0:1789761778.039360:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024624384:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789761778.039397:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789761778.039408:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761778/real 1789761778] req@ffffa06c1872f100 x1876700024624384/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761803 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761778.039419:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872f100 x1876700024624384/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761803 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761778.039427:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761778.039430:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789761778.039433:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024624384:192.168.203.107@tcp:38:kworker.0 00000100:00100000:2.0:1789761778.039461:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024624512:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789761778.039480:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789761778.039491:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761778/real 1789761778] req@ffffa06c1872ca80 x1876700024624512/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761803 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761778.039499:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872ca80 x1876700024624512/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761803 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761778.039505:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761778.039507:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789761778.039509:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024624512:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789761778.294966:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761778.550995:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761779.319149:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761779.574982:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761780.342951:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761780.598941:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761781.366981:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761781.622963:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761782.391039:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761782.646941:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789761783.158986:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761783.158996:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789761783.158999:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761783.159009:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1216 00000100:00080000:2.0:1789761783.159020:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761783.159047:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789761783.159085:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e52300 x1876700024624640/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761783.159109:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761783.159112:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761783.159113:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761783.159117:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1216 00000100:00080000:2.0:1789761783.159120:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761783.159123:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789761783.159132:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761783.159141:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e52300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024624640:192.168.203.107@tcp:250:kworker.0 00000100:00100000:2.0:1789761783.159143:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51880 x1876700024624768/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761783.159156:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761783.159159:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761783.159160:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761783.159163:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1216 00000100:00080000:2.0:1789761783.159166:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761783.159171:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789761783.159175:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761783.159188:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51180 x1876700024624896/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789761783.159226:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789761783.159271:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761783/real 1789761783] req@ffffa06c07e52300 x1876700024624640/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761808 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761783.159285:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e52300 x1876700024624640/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761808 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761783.159292:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761783.159295:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789761783.159298:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e52300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024624640:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789761783.159319:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024624768:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789761783.159356:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789761783.159371:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761783/real 1789761783] req@ffffa06c07e51880 x1876700024624768/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761808 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761783.159392:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51880 x1876700024624768/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761808 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761783.159401:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761783.159403:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789761783.159405:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024624768:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789761783.159409:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024624896:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789761783.159425:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789761783.159433:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761783/real 1789761783] req@ffffa06c07e51180 x1876700024624896/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761808 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761783.159440:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51180 x1876700024624896/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761808 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761783.159445:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761783.159447:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789761783.159449:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024624896:192.168.203.107@tcp:38:kworker.0 00010000:00010000:1.0:1789761783.414979:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761783.670953:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761784.438991:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761784.694911:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761785.462993:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761785.718950:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761786.486956:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761786.742910:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761787.510969:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761787.766939:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789761788.279004:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761788.279013:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761788.279017:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761788.279028:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1222 00000100:00080000:0.0:1789761788.279039:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761788.279046:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789761788.279090:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d1f80 x1876700024625024/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761788.279118:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761788.279121:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761788.279122:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761788.279125:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1222 00000100:00080000:0.0:1789761788.279128:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761788.279131:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789761788.279140:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761788.279155:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0a80 x1876700024625152/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761788.279184:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761788.279189:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:3.0:1789761788.279191:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024625024:192.168.203.107@tcp:250:kworker.0 00000100:00080000:0.0:1789761788.279192:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761788.279204:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1222 00000100:00080000:0.0:1789761788.279208:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761788.279211:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789761788.279216:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761788.279230:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0700 x1876700024625280/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789761788.279258:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789761788.279305:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761788/real 1789761788] req@ffffa06c091d1f80 x1876700024625024/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761813 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761788.279319:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d1f80 x1876700024625024/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761813 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761788.279342:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761788.279346:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789761788.279348:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024625024:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789761788.279375:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024625152:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789761788.279393:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789761788.279401:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761788/real 1789761788] req@ffffa06c091d0a80 x1876700024625152/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761813 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761788.279408:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0a80 x1876700024625152/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761813 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761788.279413:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761788.279415:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789761788.279417:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024625152:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789761788.279421:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024625280:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789761788.279431:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789761788.279444:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761788/real 1789761788] req@ffffa06c091d0700 x1876700024625280/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761813 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761788.279451:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0700 x1876700024625280/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761813 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761788.279455:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761788.279457:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789761788.279459:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024625280:192.168.203.107@tcp:38:kworker.0 00010000:00010000:1.0:1789761788.535030:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761788.790967:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761789.559034:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761789.814924:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761790.582959:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761790.838950:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761791.606968:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761791.862951:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761792.630944:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761792.886970:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:1.0:1789761793.398924:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761793.398932:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789761793.398942:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761793.398950:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1227 00000100:00080000:1.0:1789761793.398958:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761793.398963:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789761793.398995:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2d180 x1876700024625408/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761793.399016:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761793.399020:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761793.399021:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761793.399024:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1227 00000100:00080000:1.0:1789761793.399026:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761793.399028:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789761793.399035:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761793.399047:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2ca80 x1876700024625536/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761793.399056:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761793.399071:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761793.399074:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1789761793.399078:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024625408:192.168.203.107@tcp:250:kworker.0 00000100:00080000:1.0:1789761793.399084:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1227 00000100:00080000:1.0:1789761793.399088:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761793.399095:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789761793.399101:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761793.399116:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2c700 x1876700024625664/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789761793.399174:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789761793.399223:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761793/real 1789761793] req@ffffa06c08e2d180 x1876700024625408/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761818 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761793.399239:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2d180 x1876700024625408/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761818 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761793.399248:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761793.399252:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789761793.399267:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024625408:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789761793.399276:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024625536:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789761793.399297:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789761793.399307:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761793/real 1789761793] req@ffffa06c08e2ca80 x1876700024625536/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761818 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761793.399315:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2ca80 x1876700024625536/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761818 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761793.399322:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761793.399324:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789761793.399327:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024625536:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789761793.399349:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024625664:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789761793.399361:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789761793.399368:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761793/real 1789761793] req@ffffa06c08e2c700 x1876700024625664/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761818 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761793.399389:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2c700 x1876700024625664/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761818 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761793.399395:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761793.399397:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789761793.399399:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024625664:192.168.203.107@tcp:38:kworker.0 00010000:00010000:1.0:1789761793.654991:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761793.910962:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761794.678947:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761794.934932:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761795.702964:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761795.958944:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761796.726992:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761796.983008:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761797.750956:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761798.006954:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789761798.518995:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761798.519005:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789761798.519031:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761798.519041:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1232 00000100:00080000:2.0:1789761798.519053:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761798.519060:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789761798.519102:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51180 x1876700024625792/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761798.519127:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761798.519130:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761798.519131:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761798.519135:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1232 00000100:00080000:2.0:1789761798.519137:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761798.519140:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789761798.519149:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761798.519159:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51880 x1876700024625920/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761798.519169:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761798.519172:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761798.519173:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761798.519175:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1232 00000100:00080000:2.0:1789761798.519177:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761798.519183:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:3.0:1789761798.519183:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024625792:192.168.203.107@tcp:250:kworker.0 00000008:00100000:2.0:1789761798.519188:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761798.519196:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e52300 x1876700024626048/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789761798.519277:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789761798.519320:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761798/real 1789761798] req@ffffa06c07e51180 x1876700024625792/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761823 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761798.519337:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51180 x1876700024625792/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761823 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761798.519346:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761798.519349:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789761798.519352:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024625792:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789761798.519358:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024625920:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789761798.519374:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789761798.519382:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761798/real 1789761798] req@ffffa06c07e51880 x1876700024625920/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761823 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761798.519391:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51880 x1876700024625920/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761823 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761798.519408:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761798.519411:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789761798.519413:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024625920:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789761798.519436:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e52300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024626048:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789761798.519451:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789761798.519458:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761798/real 1789761798] req@ffffa06c07e52300 x1876700024626048/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761823 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761798.519466:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e52300 x1876700024626048/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761823 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761798.519471:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761798.519474:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789761798.519476:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e52300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024626048:192.168.203.107@tcp:38:kworker.0 00010000:00010000:1.0:1789761798.774991:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761799.030990:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761799.798965:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761800.054939:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761800.822974:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761801.078960:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761801.847018:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761802.102955:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761802.871016:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761803.126953:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:1.0:1789761803.638962:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761803.638977:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789761803.639004:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761803.639015:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1237 00000100:00080000:1.0:1789761803.639028:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761803.639035:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789761803.639075:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2ed80 x1876700024626176/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761803.639102:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761803.639105:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761803.639106:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761803.639110:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1237 00000100:00080000:1.0:1789761803.639112:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761803.639115:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789761803.639125:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761803.639138:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2c000 x1876700024626304/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761803.639152:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761803.639156:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761803.639159:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761803.639164:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1237 00000100:00080000:1.0:1789761803.639168:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1789761803.639177:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024626176:192.168.203.107@tcp:250:kworker.0 00000100:00080000:1.0:1789761803.639197:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789761803.639212:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761803.639230:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2e300 x1876700024626432/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789761803.639272:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789761803.639316:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761803/real 1789761803] req@ffffa06c08e2ed80 x1876700024626176/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761828 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761803.639330:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2ed80 x1876700024626176/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761828 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761803.639338:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761803.639354:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789761803.639357:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024626176:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789761803.639364:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024626304:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789761803.639382:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789761803.639390:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761803/real 1789761803] req@ffffa06c08e2c000 x1876700024626304/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761828 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761803.639398:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2c000 x1876700024626304/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761828 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761803.639404:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761803.639406:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789761803.639408:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024626304:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789761803.639428:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024626432:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789761803.639438:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789761803.639445:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761803/real 1789761803] req@ffffa06c08e2e300 x1876700024626432/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761828 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761803.639452:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2e300 x1876700024626432/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761828 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761803.639457:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761803.639459:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789761803.639461:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024626432:192.168.203.107@tcp:38:kworker.0 00010000:00010000:1.0:1789761803.894969:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761804.150926:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761804.919006:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761805.174936:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761805.942975:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761806.198922:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761806.966964:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761807.222951:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761807.990986:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761808.246948:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789761808.758943:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761808.758953:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789761808.758957:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761808.758967:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1242 00000100:00080000:2.0:1789761808.758978:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761808.758985:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789761808.759022:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50000 x1876700024626560/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761808.759048:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761808.759052:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761808.759054:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761808.759058:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1242 00000100:00080000:2.0:1789761808.759062:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761808.759067:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789761808.759076:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761808.759084:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024626560:192.168.203.107@tcp:250:kworker.0 00000100:00100000:2.0:1789761808.759091:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51500 x1876700024626688/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761808.759105:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761808.759108:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761808.759109:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761808.759112:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1242 00000100:00080000:2.0:1789761808.759130:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761808.759134:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789761808.759138:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1789761808.759138:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789761808.759163:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50e00 x1876700024626816/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761808.759195:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761808/real 1789761808] req@ffffa06c07e50000 x1876700024626560/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761833 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761808.759210:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50000 x1876700024626560/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761833 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761808.759217:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761808.759220:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789761808.759223:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024626560:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789761808.759241:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024626688:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789761808.759258:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789761808.759266:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761808/real 1789761808] req@ffffa06c07e51500 x1876700024626688/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761833 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761808.759274:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51500 x1876700024626688/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761833 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761808.759280:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761808.759282:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789761808.759284:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024626688:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789761808.759288:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024626816:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789761808.759299:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789761808.759305:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761808/real 1789761808] req@ffffa06c07e50e00 x1876700024626816/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761833 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761808.759343:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50e00 x1876700024626816/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761833 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761808.759352:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761808.759356:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789761808.759359:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024626816:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789761809.014964:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761809.270920:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761810.038970:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761810.295015:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761811.063043:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761811.319010:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761812.086995:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761812.342973:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761813.110975:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761813.366946:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789761813.879017:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761813.879026:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761813.879031:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761813.879043:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1247 00000100:00080000:0.0:1789761813.879055:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761813.879063:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789761813.879122:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0700 x1876700024626944/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761813.879158:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761813.879162:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761813.879165:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761813.879170:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1247 00000100:00080000:0.0:1789761813.879174:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761813.879178:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789761813.879194:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761813.879206:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0a80 x1876700024627072/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761813.879217:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761813.879223:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761813.879225:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1789761813.879226:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024626944:192.168.203.107@tcp:250:kworker.0 00000100:00080000:0.0:1789761813.879238:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1247 00000100:00080000:0.0:1789761813.879249:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761813.879253:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789761813.879257:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761813.879265:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d1f80 x1876700024627200/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789761813.879349:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789761813.879406:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761813/real 1789761813] req@ffffa06c091d0700 x1876700024626944/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761838 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761813.879437:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0700 x1876700024626944/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761838 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761813.879452:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761813.879459:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789761813.879465:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024626944:192.168.203.107@tcp:250:kworker.0 00000100:00100000:2.0:1789761813.879500:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024627072:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789761813.879535:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789761813.879562:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761813/real 1789761813] req@ffffa06c091d0a80 x1876700024627072/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761838 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761813.879580:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0a80 x1876700024627072/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761838 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761813.879617:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761813.879622:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789761813.879626:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024627072:192.168.203.107@tcp:38:kworker.0 00000100:00100000:2.0:1789761813.879632:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024627200:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789761813.879652:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789761813.879661:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761813/real 1789761813] req@ffffa06c091d1f80 x1876700024627200/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761838 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761813.879670:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d1f80 x1876700024627200/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761838 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761813.879675:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761813.879678:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789761813.879680:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024627200:192.168.203.107@tcp:38:kworker.0 00010000:00010000:2.0:1789761814.134990:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761814.390989:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761815.158946:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761815.414947:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789761816.183051:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761816.439000:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761817.206965:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761817.462941:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761818.230983:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761818.486989:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789761818.998951:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761818.998963:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789761818.998967:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761818.998977:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1252 00000100:00080000:2.0:1789761818.998988:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761818.998994:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789761818.999024:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e53b80 x1876700024627328/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761818.999060:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761818.999075:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761818.999077:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761818.999082:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1252 00000100:00080000:2.0:1789761818.999086:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761818.999089:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789761818.999097:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761818.999109:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e53b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024627328:192.168.203.107@tcp:250:kworker.0 00000100:00100000:2.0:1789761818.999110:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50380 x1876700024627456/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761818.999127:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761818.999130:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761818.999131:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761818.999134:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1252 00000100:00080000:2.0:1789761818.999137:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761818.999139:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789761818.999142:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761818.999151:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51f80 x1876700024627584/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761818.999204:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024627456:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789761818.999229:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024627584:192.168.203.107@tcp:38:kworker.0 00010000:00010000:0.0:1789761819.254949:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761819.510928:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761820.278989:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761820.534977:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761821.302968:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761821.558922:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000400:00000100:0.0:1789761822.071019:0:2329:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.107:988 00000400:00000100:0.0:1789761822.071070:0:2329:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.107@tcp at host 192.168.203.107 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1789761822.071087:0:2329:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.107@tcp 00000800:00000100:0.0:1789761822.071123:0:2329:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.107@tcp 00000800:00000100:0.0:1789761822.071135:0:2329:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.107@tcp 00000100:00100000:3.0:1789761822.071150:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761818/real 1789761822] req@ffffa06c07e50380 x1876700024627456/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761843 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761822.071178:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50380 x1876700024627456/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761843 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761822.071190:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761822.071196:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789761822.071200:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024627456:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789761822.071222:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761818/real 1789761822] req@ffffa06c07e51f80 x1876700024627584/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761843 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761822.071238:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51f80 x1876700024627584/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761843 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761822.071247:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761822.071251:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789761822.071254:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024627584:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789761822.071262:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761818/real 1789761822] req@ffffa06c07e53b80 x1876700024627328/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761843 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761822.071276:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e53b80 x1876700024627328/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761843 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761822.071284:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761822.071288:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789761822.071292:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e53b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024627328:192.168.203.107@tcp:250:kworker.0 00010000:00010000:0.0:1789761822.326985:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761822.582953:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789761823.351078:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761823.606924:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789761824.118985:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761824.118996:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761824.119000:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761824.119010:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1257 00000100:00080000:0.0:1789761824.119045:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761824.119054:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789761824.119098:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0380 x1876700024627712/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761824.119131:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761824.119136:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761824.119138:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761824.119143:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1257 00000100:00080000:0.0:1789761824.119149:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761824.119166:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:3.0:1789761824.119173:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024627712:192.168.203.107@tcp:250:kworker.0 00000008:00100000:0.0:1789761824.119184:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761824.119203:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d2680 x1876700024627840/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761824.119218:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761824.119223:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761824.119225:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761824.119230:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1257 00000100:00080000:0.0:1789761824.119235:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761824.119240:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789761824.119246:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761824.119261:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d3100 x1876700024627968/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789761824.119272:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789761824.119329:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761824/real 1789761824] req@ffffa06c091d0380 x1876700024627712/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761849 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761824.119358:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0380 x1876700024627712/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761849 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761824.119372:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761824.119386:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789761824.119391:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024627712:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789761824.119421:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024627840:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789761824.119450:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789761824.119466:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761824/real 1789761824] req@ffffa06c091d2680 x1876700024627840/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761849 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761824.119481:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d2680 x1876700024627840/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761849 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761824.119504:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761824.119508:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789761824.119512:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024627840:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789761824.119519:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024627968:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789761824.119549:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789761824.119564:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761824/real 1789761824] req@ffffa06c091d3100 x1876700024627968/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761849 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761824.119577:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d3100 x1876700024627968/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761849 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761824.119755:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761824.119759:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789761824.119764:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024627968:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789761824.375000:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761824.630940:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761825.399067:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761825.655045:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789761826.422981:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761826.678950:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761827.446976:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761827.702961:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761828.470973:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761828.726956:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:1.0:1789761829.238939:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761829.238947:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789761829.238950:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761829.238974:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1263 00000100:00080000:1.0:1789761829.238983:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761829.239002:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789761829.239032:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2d500 x1876700024628096/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761829.239053:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761829.239056:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761829.239057:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761829.239060:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1263 00000100:00080000:1.0:1789761829.239063:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761829.239066:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789761829.239073:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761829.239084:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2e300 x1876700024628224/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761829.239089:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024628096:192.168.203.107@tcp:250:kworker.0 00000100:00080000:1.0:1789761829.239096:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761829.239100:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761829.239101:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761829.239104:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1263 00000100:00080000:1.0:1789761829.239106:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761829.239109:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789761829.239112:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761829.239120:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2c000 x1876700024628352/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789761829.239148:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789761829.239183:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761829/real 1789761829] req@ffffa06c08e2d500 x1876700024628096/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761854 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761829.239196:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2d500 x1876700024628096/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761854 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761829.239203:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761829.239206:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789761829.239209:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024628096:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789761829.239226:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024628224:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789761829.239240:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789761829.239248:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761829/real 1789761829] req@ffffa06c08e2e300 x1876700024628224/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761854 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761829.239256:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2e300 x1876700024628224/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761854 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761829.239262:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761829.239264:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789761829.239267:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024628224:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789761829.239270:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024628352:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789761829.239315:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789761829.239328:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761829/real 1789761829] req@ffffa06c08e2c000 x1876700024628352/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761854 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761829.239340:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2c000 x1876700024628352/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761854 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761829.239345:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761829.239348:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789761829.239350:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024628352:192.168.203.107@tcp:38:kworker.0 00010000:00010000:0.0:1789761829.495075:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761829.750971:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761830.518974:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761830.774953:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761831.542995:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761831.798996:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789761832.566948:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761832.822906:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761833.591010:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761833.846975:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789761834.358932:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761834.358941:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761834.358945:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761834.358970:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1268 00000100:00080000:0.0:1789761834.358979:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761834.358984:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789761834.359015:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d1500 x1876700024628480/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761834.359036:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761834.359038:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761834.359040:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761834.359043:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1268 00000100:00080000:0.0:1789761834.359045:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761834.359048:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789761834.359056:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761834.359066:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d2a00 x1876700024628608/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761834.359075:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761834.359078:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761834.359081:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1789761834.359088:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024628480:192.168.203.107@tcp:250:kworker.0 00000100:00080000:0.0:1789761834.359089:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1268 00000100:00080000:0.0:1789761834.359098:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761834.359107:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789761834.359127:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761834.359140:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0e00 x1876700024628736/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789761834.359178:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789761834.359227:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761834/real 1789761834] req@ffffa06c091d1500 x1876700024628480/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761859 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761834.359244:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d1500 x1876700024628480/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761859 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761834.359253:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761834.359257:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789761834.359260:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024628480:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789761834.359284:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024628608:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789761834.359300:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789761834.359325:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761834/real 1789761834] req@ffffa06c091d2a00 x1876700024628608/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761859 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761834.359334:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d2a00 x1876700024628608/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761859 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761834.359341:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761834.359343:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789761834.359346:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024628608:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789761834.359350:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024628736:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789761834.359364:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789761834.359373:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761834/real 1789761834] req@ffffa06c091d0e00 x1876700024628736/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761859 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761834.359387:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0e00 x1876700024628736/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761859 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761834.359395:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761834.359399:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789761834.359402:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024628736:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789761834.614993:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761834.870982:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789761835.639041:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761835.894961:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761836.662982:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761836.918926:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761837.686992:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761837.942964:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789761838.710992:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761838.966941:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:1.0:1789761839.479003:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761839.479014:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789761839.479018:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761839.479029:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1273 00000100:00080000:1.0:1789761839.479042:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761839.479051:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789761839.479089:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2ed80 x1876700024628864/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761839.479117:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761839.479121:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761839.479123:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761839.479127:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1273 00000100:00080000:1.0:1789761839.479131:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761839.479135:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789761839.479145:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761839.479161:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024628864:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789761839.479180:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2c700 x1876700024628992/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761839.479196:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761839.479200:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761839.479202:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761839.479206:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1273 00000100:00080000:1.0:1789761839.479211:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761839.479216:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789761839.479222:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1789761839.479227:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761839.479236:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2ca80 x1876700024629120/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761839.479285:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761839/real 1789761839] req@ffffa06c08e2ed80 x1876700024628864/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761864 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761839.479318:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2ed80 x1876700024628864/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761864 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761839.479325:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761839.479329:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789761839.479332:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024628864:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789761839.479351:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024628992:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789761839.479377:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789761839.479387:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761839/real 1789761839] req@ffffa06c08e2c700 x1876700024628992/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761864 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761839.479397:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2c700 x1876700024628992/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761864 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761839.479406:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761839.479410:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789761839.479413:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024628992:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789761839.479419:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024629120:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789761839.479440:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789761839.479452:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761839/real 1789761839] req@ffffa06c08e2ca80 x1876700024629120/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761864 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761839.479462:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2ca80 x1876700024629120/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761864 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761839.479486:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761839.479490:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789761839.479494:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024629120:192.168.203.107@tcp:38:kworker.0 00010000:00010000:2.0:1789761839.735081:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761839.990957:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761840.759085:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761841.014950:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789761841.783078:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761842.038971:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761842.807013:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761843.062939:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761843.830986:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761844.086935:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789761844.598964:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761844.598973:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789761844.598989:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761844.598997:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1278 00000100:00080000:2.0:1789761844.599005:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761844.599011:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789761844.599039:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e52680 x1876700024629248/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761844.599059:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761844.599062:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761844.599063:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761844.599066:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1278 00000100:00080000:2.0:1789761844.599068:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761844.599070:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789761844.599077:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761844.599086:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e53800 x1876700024629376/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761844.599095:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761844.599098:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761844.599099:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1789761844.599104:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e52680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024629248:192.168.203.107@tcp:250:kworker.0 00000100:00080000:2.0:1789761844.599105:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1278 00000100:00080000:2.0:1789761844.599110:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761844.599116:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789761844.599119:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761844.599126:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51c00 x1876700024629504/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789761844.599164:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789761844.599199:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761844/real 1789761844] req@ffffa06c07e52680 x1876700024629248/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761869 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761844.599211:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e52680 x1876700024629248/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761869 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761844.599219:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761844.599222:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789761844.599225:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e52680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024629248:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789761844.599242:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e53800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024629376:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789761844.599255:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789761844.599263:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761844/real 1789761844] req@ffffa06c07e53800 x1876700024629376/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761869 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761844.599270:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e53800 x1876700024629376/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761869 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761844.599287:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761844.599296:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789761844.599299:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e53800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024629376:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789761844.599303:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024629504:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789761844.599319:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789761844.599330:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761844/real 1789761844] req@ffffa06c07e51c00 x1876700024629504/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761869 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761844.599342:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51c00 x1876700024629504/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761869 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761844.599350:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761844.599354:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789761844.599357:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024629504:192.168.203.107@tcp:38:kworker.0 00010000:00010000:0.0:1789761844.854980:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761845.110919:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761845.878970:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761846.134976:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761846.902994:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761847.158953:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761847.927083:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761848.182996:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761848.951007:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761849.207030:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789761849.718964:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761849.718987:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789761849.718992:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761849.719006:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1283 00000100:00080000:2.0:1789761849.719020:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761849.719029:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789761849.719083:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50a80 x1876700024629632/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761849.719115:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761849.719119:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761849.719121:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761849.719127:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1283 00000100:00080000:2.0:1789761849.719132:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761849.719138:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789761849.719150:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761849.719170:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50700 x1876700024629760/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789761849.719197:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024629632:192.168.203.107@tcp:250:kworker.0 00000100:00080000:2.0:1789761849.719198:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761849.719207:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761849.719209:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761849.719218:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1283 00000100:00080000:2.0:1789761849.719222:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761849.719250:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789761849.719255:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761849.719265:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e53480 x1876700024629888/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789761849.719288:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761849.719354:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761849/real 1789761849] req@ffffa06c07e50a80 x1876700024629632/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761874 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761849.719385:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50a80 x1876700024629632/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761874 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761849.719399:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761849.719403:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761849.719407:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024629632:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789761849.719434:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024629760:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761849.719469:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761849.719484:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761849/real 1789761849] req@ffffa06c07e50700 x1876700024629760/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761874 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761849.719500:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50700 x1876700024629760/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761874 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761849.719507:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761849.719510:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761849.719513:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024629760:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789761849.719529:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e53480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024629888:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761849.719547:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761849.719557:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761849/real 1789761849] req@ffffa06c07e53480 x1876700024629888/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761874 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761849.719566:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e53480 x1876700024629888/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761874 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761849.719583:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761849.719587:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761849.719591:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e53480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024629888:192.168.203.107@tcp:38:kworker.0 00010000:00010000:2.0:1789761849.975002:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761850.230996:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761850.998968:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761851.254951:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761852.023044:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761852.278990:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761853.047024:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761853.303008:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761854.070998:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761854.326935:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789761854.839069:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761854.839084:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789761854.839089:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761854.839103:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1288 00000100:00080000:2.0:1789761854.839117:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761854.839127:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789761854.839196:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e52a00 x1876700024630016/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761854.839247:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761854.839252:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761854.839254:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761854.839260:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1288 00000100:00080000:2.0:1789761854.839264:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761854.839269:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789761854.839284:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761854.839316:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e53100 x1876700024630144/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761854.839331:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761854.839335:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:1.0:1789761854.839343:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e52a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024630016:192.168.203.107@tcp:250:kworker.0 00000100:00080000:2.0:1789761854.839375:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761854.839397:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1288 00000100:00080000:2.0:1789761854.839404:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761854.839412:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000800:00000100:1.0:1789761854.839421:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000008:00100000:2.0:1789761854.839422:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761854.839446:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e52d80 x1876700024630272/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789761854.839460:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761854/real 1789761854] req@ffffa06c07e52a00 x1876700024630016/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761879 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761854.839489:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e52a00 x1876700024630016/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761879 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761854.839498:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761854.839502:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761854.839505:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e52a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024630016:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789761854.839524:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e53100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024630144:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761854.839550:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761854.839561:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761854/real 1789761854] req@ffffa06c07e53100 x1876700024630144/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761879 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761854.839590:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e53100 x1876700024630144/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761879 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761854.839600:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761854.839604:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761854.839608:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e53100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024630144:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789761854.839626:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e52d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024630272:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761854.839643:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761854.839651:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761854/real 1789761854] req@ffffa06c07e52d80 x1876700024630272/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761879 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761854.839659:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e52d80 x1876700024630272/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761879 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761854.839665:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761854.839667:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761854.839669:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e52d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024630272:192.168.203.107@tcp:38:kworker.0 00010000:00010000:1.0:1789761855.094983:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761855.350941:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761856.118952:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761856.374953:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761857.143024:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761857.398955:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761858.166985:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761858.422912:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761859.190979:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761859.446955:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789761859.958977:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761859.958989:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789761859.958993:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761859.959004:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1293 00000100:00080000:2.0:1789761859.959016:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761859.959023:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789761859.959071:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e52d80 x1876700024630400/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761859.959099:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761859.959102:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761859.959103:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761859.959107:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1293 00000100:00080000:2.0:1789761859.959109:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761859.959112:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789761859.959120:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761859.959130:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e53100 x1876700024630528/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761859.959170:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1789761859.959178:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e52d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024630400:192.168.203.107@tcp:250:kworker.0 00000100:00080000:2.0:1789761859.959179:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761859.959182:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761859.959190:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1293 00000100:00080000:2.0:1789761859.959196:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761859.959209:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789761859.959217:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761859.959231:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e52a00 x1876700024630656/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789761859.959264:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761859.959302:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761859/real 1789761859] req@ffffa06c07e52d80 x1876700024630400/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761884 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761859.959316:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e52d80 x1876700024630400/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761884 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761859.959325:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761859.959328:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761859.959331:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e52d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024630400:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789761859.959355:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e53100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024630528:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761859.959378:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761859.959392:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761859/real 1789761859] req@ffffa06c07e53100 x1876700024630528/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761884 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761859.959405:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e53100 x1876700024630528/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761884 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761859.959411:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761859.959413:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761859.959415:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e53100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024630528:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789761859.959420:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e52a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024630656:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761859.959435:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761859.959442:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761859/real 1789761859] req@ffffa06c07e52a00 x1876700024630656/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761884 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761859.959460:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e52a00 x1876700024630656/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761884 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761859.959465:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761859.959467:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761859.959469:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e52a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024630656:192.168.203.107@tcp:38:kworker.0 00010000:00010000:2.0:1789761860.215145:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761860.470954:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761861.238983:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761861.494937:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761862.262981:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761862.518910:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761863.286988:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761863.542974:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761864.311058:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761864.566903:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:3.0:1789761865.078957:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761865.078964:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789761865.078968:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761865.078976:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1298 00000100:00080000:3.0:1789761865.078985:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761865.078991:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:3.0:1789761865.079036:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872ce00 x1876700024630784/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761865.079061:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761865.079064:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761865.079065:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761865.079068:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1298 00000100:00080000:3.0:1789761865.079071:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761865.079074:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789761865.079081:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761865.079091:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872d180 x1876700024630912/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761865.079100:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761865.079105:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:1.0:1789761865.079106:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024630784:192.168.203.107@tcp:250:kworker.0 00000100:00080000:3.0:1789761865.079106:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761865.079114:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1298 00000100:00080000:3.0:1789761865.079117:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761865.079119:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789761865.079122:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761865.079130:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872e300 x1876700024631040/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789761865.079171:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761865.079210:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761865/real 1789761865] req@ffffa06c1872ce00 x1876700024630784/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761890 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761865.079229:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872ce00 x1876700024630784/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761890 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761865.079240:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761865.079245:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761865.079249:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024630784:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789761865.079272:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024630912:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761865.079297:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761865.079321:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761865/real 1789761865] req@ffffa06c1872d180 x1876700024630912/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761890 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761865.079335:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872d180 x1876700024630912/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761890 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761865.079343:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761865.079347:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761865.079351:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024630912:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789761865.079357:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024631040:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761865.079379:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761865.079391:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761865/real 1789761865] req@ffffa06c1872e300 x1876700024631040/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761890 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761865.079403:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872e300 x1876700024631040/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761890 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761865.079412:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761865.079416:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761865.079419:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024631040:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789761865.334975:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761865.590963:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761866.358996:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761866.614939:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761867.382966:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761867.638958:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761868.407055:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761868.662901:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761869.430970:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761869.686935:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:3.0:1789761870.199005:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761870.199052:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789761870.199055:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761870.199065:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1304 00000100:00080000:3.0:1789761870.199076:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761870.199083:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:3.0:1789761870.199138:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872d880 x1876700024631168/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761870.199164:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761870.199169:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761870.199172:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761870.199178:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1304 00000100:00080000:3.0:1789761870.199183:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761870.199189:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789761870.199200:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761870.199218:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872c380 x1876700024631296/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761870.199234:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1789761870.199236:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024631168:192.168.203.107@tcp:250:kworker.0 00000100:00080000:3.0:1789761870.199239:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761870.199241:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761870.199246:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1304 00000100:00080000:3.0:1789761870.199251:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761870.199256:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789761870.199262:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761870.199275:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872df80 x1876700024631424/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789761870.199305:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761870.199355:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761870/real 1789761870] req@ffffa06c1872d880 x1876700024631168/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761895 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761870.199375:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872d880 x1876700024631168/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761895 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761870.199419:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761870.199422:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761870.199426:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024631168:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789761870.199446:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024631296:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761870.199487:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761870.199499:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761870/real 1789761870] req@ffffa06c1872c380 x1876700024631296/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761895 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761870.199509:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872c380 x1876700024631296/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761895 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761870.199517:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761870.199520:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761870.199523:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024631296:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789761870.199528:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024631424:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761870.199541:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761870.199549:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761870/real 1789761870] req@ffffa06c1872df80 x1876700024631424/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761895 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761870.199583:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872df80 x1876700024631424/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761895 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761870.199590:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761870.199593:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761870.199595:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024631424:192.168.203.107@tcp:38:kworker.0 00010000:00010000:1.0:1789761870.454990:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761870.710941:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761871.478945:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761871.734954:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761872.502995:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761872.758905:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761873.527012:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761873.782974:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761874.551032:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761874.807025:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789761875.318986:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761875.318994:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789761875.318997:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761875.319004:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1309 00000100:00080000:2.0:1789761875.319013:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761875.319019:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789761875.319052:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e53480 x1876700024631552/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761875.319074:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761875.319077:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761875.319079:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761875.319081:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1309 00000100:00080000:2.0:1789761875.319097:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761875.319100:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789761875.319107:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761875.319117:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50700 x1876700024631680/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761875.319124:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761875.319127:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761875.319128:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761875.319130:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1309 00000100:00080000:2.0:1789761875.319132:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761875.319135:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789761875.319137:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761875.319144:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50a80 x1876700024631808/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789761875.319195:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e53480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024631552:192.168.203.107@tcp:250:kworker.0 00000800:00000100:1.0:1789761875.319277:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761875.319325:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761875/real 1789761875] req@ffffa06c07e53480 x1876700024631552/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761900 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761875.319340:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e53480 x1876700024631552/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761900 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761875.319350:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761875.319354:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761875.319357:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e53480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024631552:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789761875.319362:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024631680:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761875.319392:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761875.319401:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761875/real 1789761875] req@ffffa06c07e50700 x1876700024631680/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761900 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761875.319409:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50700 x1876700024631680/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761900 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761875.319415:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761875.319418:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761875.319420:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024631680:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789761875.319424:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024631808:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761875.319434:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761875.319440:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761875/real 1789761875] req@ffffa06c07e50a80 x1876700024631808/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761900 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761875.319447:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50a80 x1876700024631808/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761900 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761875.319453:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761875.319454:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761875.319456:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024631808:192.168.203.107@tcp:38:kworker.0 00010000:00010000:0.0:1789761875.575003:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761875.830908:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761876.598983:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761876.854966:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761877.622947:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761877.878924:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789761878.647047:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761878.902974:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761879.670931:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761879.926922:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789761880.438918:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761880.438926:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789761880.438929:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761880.438937:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1314 00000100:00080000:2.0:1789761880.438946:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761880.438951:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789761880.438982:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51c00 x1876700024631936/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761880.439003:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761880.439019:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761880.439021:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761880.439024:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1314 00000100:00080000:2.0:1789761880.439027:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761880.439030:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789761880.439042:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761880.439051:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e53800 x1876700024632064/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761880.439060:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761880.439063:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761880.439065:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761880.439068:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1314 00000100:00080000:2.0:1789761880.439070:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761880.439072:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789761880.439075:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761880.439081:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e52680 x1876700024632192/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789761880.439085:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024631936:192.168.203.107@tcp:250:kworker.0 00000800:00000100:1.0:1789761880.439140:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761880.439177:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761880/real 1789761880] req@ffffa06c07e51c00 x1876700024631936/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761905 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761880.439197:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51c00 x1876700024631936/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761905 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761880.439209:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761880.439214:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761880.439218:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024631936:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789761880.439226:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e53800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024632064:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761880.439243:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761880.439251:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761880/real 1789761880] req@ffffa06c07e53800 x1876700024632064/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761905 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761880.439261:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e53800 x1876700024632064/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761905 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761880.439270:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761880.439273:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761880.439276:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e53800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024632064:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789761880.439300:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e52680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024632192:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761880.439319:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761880.439327:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761880/real 1789761880] req@ffffa06c07e52680 x1876700024632192/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761905 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761880.439343:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e52680 x1876700024632192/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761905 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761880.439348:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761880.439350:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761880.439352:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e52680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024632192:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789761880.694958:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761880.950944:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761881.718978:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761881.974912:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761882.742965:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761882.998947:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761883.767004:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761884.022947:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761884.791011:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761885.047029:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789761885.558994:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761885.559012:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789761885.559018:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761885.559031:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1319 00000100:00080000:2.0:1789761885.559046:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761885.559055:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789761885.559131:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e53b80 x1876700024632320/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761885.559173:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761885.559179:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761885.559183:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761885.559190:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1319 00000100:00080000:2.0:1789761885.559195:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761885.559201:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789761885.559213:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761885.559219:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e53b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024632320:192.168.203.107@tcp:250:kworker.0 00000100:00100000:2.0:1789761885.559234:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51f80 x1876700024632448/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761885.559252:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761885.559257:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761885.559259:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761885.559265:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1319 00000100:00080000:2.0:1789761885.559270:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761885.559276:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789761885.559283:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761885.559297:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50380 x1876700024632576/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789761885.559332:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024632448:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789761885.559368:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024632576:192.168.203.107@tcp:38:kworker.0 00010000:00010000:1.0:1789761885.815126:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761886.070960:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761886.838966:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761887.094933:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761887.863063:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761888.118935:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000400:00000100:3.0:1789761888.631104:0:2330:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.107:988 00000400:00000100:3.0:1789761888.631154:0:2330:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.107@tcp at host 192.168.203.107 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1789761888.631173:0:2330:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.107@tcp 00000800:00000100:3.0:1789761888.631217:0:2330:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.107@tcp 00000800:00000100:3.0:1789761888.631233:0:2330:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.107@tcp 00000100:00100000:1.0:1789761888.631268:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761885/real 1789761888] req@ffffa06c07e51f80 x1876700024632448/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761910 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761888.631303:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51f80 x1876700024632448/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761910 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761888.631332:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761888.631338:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761888.631343:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024632448:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789761888.631352:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761885/real 1789761888] req@ffffa06c07e50380 x1876700024632576/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761910 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761888.631367:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50380 x1876700024632576/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761910 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761888.631376:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761888.631380:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761888.631383:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024632576:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789761888.631391:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761885/real 1789761888] req@ffffa06c07e53b80 x1876700024632320/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761910 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761888.631414:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e53b80 x1876700024632320/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761910 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761888.631425:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761888.631429:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761888.631433:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e53b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024632320:192.168.203.107@tcp:250:kworker.0 00010000:00010000:1.0:1789761888.886982:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761889.142916:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761889.910986:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761890.166968:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789761890.678922:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761890.678934:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761890.678938:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761890.678949:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1324 00000100:00080000:0.0:1789761890.678969:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761890.678978:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789761890.679012:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d3b80 x1876700024632704/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761890.679034:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761890.679038:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761890.679040:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761890.679045:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1324 00000100:00080000:0.0:1789761890.679049:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761890.679054:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789761890.679064:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761890.679066:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024632704:192.168.203.107@tcp:250:kworker.0 00000100:00100000:0.0:1789761890.679081:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d2300 x1876700024632832/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761890.679105:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761890.679109:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761890.679110:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761890.679115:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1324 00000100:00080000:0.0:1789761890.679137:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761890.679142:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000800:00000100:1.0:1789761890.679148:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000008:00100000:0.0:1789761890.679151:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761890.679169:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0000 x1876700024632960/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789761890.679205:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761890/real 1789761890] req@ffffa06c091d3b80 x1876700024632704/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761915 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761890.679227:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d3b80 x1876700024632704/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761915 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761890.679239:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761890.679243:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761890.679248:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024632704:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789761890.679272:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024632832:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761890.679309:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761890.679324:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761890/real 1789761890] req@ffffa06c091d2300 x1876700024632832/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761915 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761890.679338:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d2300 x1876700024632832/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761915 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761890.679347:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761890.679351:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761890.679354:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024632832:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789761890.679361:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024632960:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761890.679384:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761890.679397:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761890/real 1789761890] req@ffffa06c091d0000 x1876700024632960/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761915 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761890.679423:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0000 x1876700024632960/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761915 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761890.679432:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761890.679435:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761890.679438:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024632960:192.168.203.107@tcp:38:kworker.0 00010000:00010000:1.0:1789761890.935110:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761891.190950:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761891.958963:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761892.214959:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761892.982952:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761893.238962:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761894.007037:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761894.263021:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761895.030997:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761895.286915:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789761895.798947:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761895.798973:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789761895.798977:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761895.798988:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1329 00000100:00080000:2.0:1789761895.798999:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761895.799007:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789761895.799048:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50e00 x1876700024633088/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761895.799068:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761895.799071:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761895.799072:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761895.799075:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1329 00000100:00080000:2.0:1789761895.799078:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761895.799081:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789761895.799088:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761895.799099:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51500 x1876700024633216/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761895.799108:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761895.799111:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761895.799112:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761895.799115:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1329 00000100:00080000:2.0:1789761895.799117:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1789761895.799123:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024633088:192.168.203.107@tcp:250:kworker.0 00000100:00080000:2.0:1789761895.799123:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789761895.799128:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761895.799138:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50000 x1876700024633344/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789761895.799202:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761895.799241:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761895/real 1789761895] req@ffffa06c07e50e00 x1876700024633088/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761920 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761895.799259:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50e00 x1876700024633088/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761920 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761895.799272:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761895.799277:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761895.799280:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024633088:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789761895.799298:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024633216:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761895.799324:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761895.799339:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761895/real 1789761895] req@ffffa06c07e51500 x1876700024633216/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761920 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761895.799353:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51500 x1876700024633216/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761920 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761895.799362:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761895.799366:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761895.799370:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024633216:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789761895.799392:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024633344:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761895.799415:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761895.799427:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761895/real 1789761895] req@ffffa06c07e50000 x1876700024633344/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761920 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761895.799439:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50000 x1876700024633344/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761920 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761895.799449:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761895.799453:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761895.799456:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024633344:192.168.203.107@tcp:38:kworker.0 00010000:00010000:2.0:1789761896.054964:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761896.310962:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761897.078935:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761897.334925:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761898.102967:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761898.358945:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761899.126971:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761899.382921:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761900.151132:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761900.407044:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:3.0:1789761900.918984:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761900.918994:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789761900.918999:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761900.919011:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1334 00000100:00080000:3.0:1789761900.919023:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761900.919030:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:3.0:1789761900.919075:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872f800 x1876700024633472/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761900.919100:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761900.919103:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761900.919108:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761900.919111:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1334 00000100:00080000:3.0:1789761900.919114:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761900.919117:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789761900.919127:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761900.919139:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872df80 x1876700024633600/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789761900.919159:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024633472:192.168.203.107@tcp:250:kworker.0 00000100:00080000:3.0:1789761900.919179:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761900.919182:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761900.919183:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761900.919186:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1334 00000100:00080000:3.0:1789761900.919190:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761900.919193:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789761900.919197:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761900.919207:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872c380 x1876700024633728/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789761900.919245:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761900.919288:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761900/real 1789761900] req@ffffa06c1872f800 x1876700024633472/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761925 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761900.919304:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872f800 x1876700024633472/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761925 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761900.919313:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761900.919316:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761900.919319:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024633472:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789761900.919342:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024633600:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761900.919357:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761900.919365:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761900/real 1789761900] req@ffffa06c1872df80 x1876700024633600/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761925 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761900.919374:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872df80 x1876700024633600/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761925 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761900.919379:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761900.919381:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761900.919383:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024633600:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789761900.919387:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024633728:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761900.919409:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761900.919416:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761900/real 1789761900] req@ffffa06c1872c380 x1876700024633728/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761925 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761900.919425:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872c380 x1876700024633728/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761925 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761900.919430:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761900.919432:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761900.919435:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024633728:192.168.203.107@tcp:38:kworker.0 00010000:00010000:0.0:1789761901.175047:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761901.430953:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761902.199000:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761902.454929:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761903.222962:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761903.478936:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761904.246962:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761904.502955:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761905.270958:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761905.526955:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:3.0:1789761906.038917:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761906.038929:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789761906.038933:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761906.038943:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1339 00000100:00080000:3.0:1789761906.038954:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761906.038961:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:3.0:1789761906.038998:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872d880 x1876700024633856/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761906.039024:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761906.039027:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761906.039028:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761906.039031:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1339 00000100:00080000:3.0:1789761906.039034:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761906.039049:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789761906.039056:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761906.039065:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872e300 x1876700024633984/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761906.039073:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761906.039075:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761906.039077:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761906.039079:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1339 00000100:00080000:3.0:1789761906.039082:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761906.039085:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789761906.039088:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761906.039095:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872d180 x1876700024634112/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789761906.039109:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024633856:192.168.203.107@tcp:250:kworker.0 00000800:00000100:1.0:1789761906.039163:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761906.039195:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761906/real 1789761906] req@ffffa06c1872d880 x1876700024633856/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761931 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761906.039208:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872d880 x1876700024633856/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761931 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761906.039219:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761906.039223:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761906.039228:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024633856:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789761906.039246:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024633984:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761906.039271:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761906.039291:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761906/real 1789761906] req@ffffa06c1872e300 x1876700024633984/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761931 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761906.039300:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872e300 x1876700024633984/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761931 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761906.039306:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761906.039309:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761906.039311:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024633984:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789761906.039328:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024634112:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761906.039342:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761906.039349:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761906/real 1789761906] req@ffffa06c1872d180 x1876700024634112/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761931 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761906.039356:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872d180 x1876700024634112/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761931 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761906.039362:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761906.039364:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761906.039366:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024634112:192.168.203.107@tcp:38:kworker.0 00010000:00010000:1.0:1789761906.295001:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761906.550959:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761907.318976:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761907.574964:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761908.342995:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761908.598927:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761909.366965:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761909.622955:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761910.390968:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761910.646931:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789761911.158955:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761911.158965:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789761911.158970:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761911.158981:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1344 00000100:00080000:2.0:1789761911.158993:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761911.159000:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789761911.159048:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e52300 x1876700024634240/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761911.159075:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761911.159078:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761911.159080:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761911.159085:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1344 00000100:00080000:2.0:1789761911.159088:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761911.159091:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789761911.159101:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761911.159114:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51880 x1876700024634368/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761911.159125:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761911.159129:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761911.159146:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1789761911.159153:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e52300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024634240:192.168.203.107@tcp:250:kworker.0 00000100:00080000:2.0:1789761911.159153:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1344 00000100:00080000:2.0:1789761911.159163:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761911.159170:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789761911.159175:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761911.159187:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51180 x1876700024634496/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789761911.159215:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761911.159248:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761911/real 1789761911] req@ffffa06c07e52300 x1876700024634240/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761936 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761911.159261:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e52300 x1876700024634240/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761936 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761911.159268:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761911.159271:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761911.159274:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e52300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024634240:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789761911.159289:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024634368:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761911.159306:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761911.159314:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761911/real 1789761911] req@ffffa06c07e51880 x1876700024634368/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761936 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761911.159322:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51880 x1876700024634368/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761936 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761911.159336:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761911.159338:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761911.159341:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024634368:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789761911.159357:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024634496:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761911.159395:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761911.159404:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761911/real 1789761911] req@ffffa06c07e51180 x1876700024634496/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761936 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761911.159412:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51180 x1876700024634496/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761936 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761911.159417:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761911.159419:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761911.159421:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024634496:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789761911.414975:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761911.670959:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761912.439042:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761912.694939:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761913.463055:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761913.718997:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761914.486956:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761914.742951:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761915.510974:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761915.766943:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789761916.278964:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761916.278973:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761916.278976:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761916.278984:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1350 00000100:00080000:0.0:1789761916.278992:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761916.278998:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789761916.279031:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0000 x1876700024634624/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761916.279053:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761916.279055:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761916.279057:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761916.279060:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1350 00000100:00080000:0.0:1789761916.279062:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761916.279077:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789761916.279085:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761916.279093:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024634624:192.168.203.107@tcp:250:kworker.0 00000100:00100000:0.0:1789761916.279097:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d2300 x1876700024634752/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761916.279107:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761916.279110:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761916.279112:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761916.279114:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1350 00000100:00080000:0.0:1789761916.279117:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761916.279120:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789761916.279123:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761916.279131:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d3b80 x1876700024634880/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789761916.279149:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761916.279182:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761916/real 1789761916] req@ffffa06c091d0000 x1876700024634624/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761941 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761916.279195:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0000 x1876700024634624/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761941 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761916.279203:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761916.279206:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761916.279209:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024634624:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789761916.279226:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024634752:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761916.279249:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761916.279258:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761916/real 1789761916] req@ffffa06c091d2300 x1876700024634752/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761941 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761916.279266:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d2300 x1876700024634752/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761941 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761916.279272:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761916.279274:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761916.279276:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024634752:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789761916.279280:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024634880:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761916.279291:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761916.279297:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761916/real 1789761916] req@ffffa06c091d3b80 x1876700024634880/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761941 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761916.279304:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d3b80 x1876700024634880/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761941 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761916.279309:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761916.279311:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761916.279313:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024634880:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789761916.534952:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761916.790923:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761917.558959:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761917.814947:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761918.582955:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761918.838925:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761919.606970:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761919.863005:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761920.630981:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761920.886909:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:3.0:1789761921.398962:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761921.398970:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789761921.398973:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761921.398995:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1355 00000100:00080000:3.0:1789761921.399006:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761921.399013:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:3.0:1789761921.399043:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872ce00 x1876700024635008/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761921.399072:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761921.399075:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761921.399076:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761921.399079:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1355 00000100:00080000:3.0:1789761921.399082:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761921.399085:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789761921.399093:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761921.399106:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872ca80 x1876700024635136/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761921.399115:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761921.399117:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761921.399119:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761921.399122:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1355 00000100:00080000:3.0:1789761921.399124:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761921.399127:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789761921.399132:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761921.399134:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024635008:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789761921.399143:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872f100 x1876700024635264/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789761921.399219:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789761921.399267:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761921/real 1789761921] req@ffffa06c1872ce00 x1876700024635008/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761946 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761921.399314:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872ce00 x1876700024635008/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761946 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761921.399330:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761921.399335:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789761921.399340:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024635008:192.168.203.107@tcp:250:kworker.0 00000100:00100000:2.0:1789761921.399349:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024635136:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789761921.399387:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789761921.399402:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761921/real 1789761921] req@ffffa06c1872ca80 x1876700024635136/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761946 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761921.399431:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872ca80 x1876700024635136/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761946 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761921.399440:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761921.399443:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789761921.399447:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024635136:192.168.203.107@tcp:38:kworker.0 00000100:00100000:2.0:1789761921.399470:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024635264:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789761921.399496:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789761921.399511:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761921/real 1789761921] req@ffffa06c1872f100 x1876700024635264/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761946 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761921.399526:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872f100 x1876700024635264/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761946 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761921.399536:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761921.399539:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789761921.399543:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024635264:192.168.203.107@tcp:38:kworker.0 00010000:00010000:2.0:1789761921.655026:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761921.910944:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761922.679068:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761922.934935:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761923.702949:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761923.958978:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761924.726975:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761924.982982:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761925.750948:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761926.006997:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:3.0:1789761926.519006:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761926.519017:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789761926.519022:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761926.519033:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1360 00000100:00080000:3.0:1789761926.519045:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761926.519054:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:3.0:1789761926.519114:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872ed80 x1876700024635392/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761926.519142:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761926.519146:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761926.519148:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761926.519152:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1360 00000100:00080000:3.0:1789761926.519156:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761926.519159:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789761926.519168:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761926.519213:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872c700 x1876700024635520/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761926.519227:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761926.519231:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761926.519233:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761926.519237:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1360 00000100:00100000:2.0:1789761926.519256:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024635392:192.168.203.107@tcp:250:kworker.0 00000100:00080000:3.0:1789761926.519298:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761926.519305:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789761926.519316:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761926.519332:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872dc00 x1876700024635648/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789761926.519415:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789761926.519493:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761926/real 1789761926] req@ffffa06c1872ed80 x1876700024635392/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761951 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761926.519553:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872ed80 x1876700024635392/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761951 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761926.519573:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761926.519581:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789761926.519588:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024635392:192.168.203.107@tcp:250:kworker.0 00000100:00100000:2.0:1789761926.519601:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024635520:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789761926.519644:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789761926.519669:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761926/real 1789761926] req@ffffa06c1872c700 x1876700024635520/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761951 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761926.519706:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872c700 x1876700024635520/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761951 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761926.519718:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761926.519724:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789761926.519729:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024635520:192.168.203.107@tcp:38:kworker.0 00000100:00100000:2.0:1789761926.519776:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024635648:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789761926.519813:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789761926.519833:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761926/real 1789761926] req@ffffa06c1872dc00 x1876700024635648/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761951 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761926.519870:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872dc00 x1876700024635648/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761951 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761926.519919:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761926.519926:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789761926.519932:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024635648:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789761926.775010:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761927.031011:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761927.799013:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761928.054985:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761928.823035:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761929.078990:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761929.846982:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761930.102970:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761930.871006:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761931.126970:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:3.0:1789761931.638930:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761931.638939:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789761931.638942:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761931.638950:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1365 00000100:00080000:3.0:1789761931.638959:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761931.638965:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:3.0:1789761931.638999:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872e680 x1876700024635776/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761931.639027:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761931.639030:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761931.639031:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761931.639034:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1365 00000100:00080000:3.0:1789761931.639036:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761931.639039:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789761931.639046:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761931.639071:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872ea00 x1876700024635904/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789761931.639080:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024635776:192.168.203.107@tcp:250:kworker.0 00000100:00080000:3.0:1789761931.639084:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761931.639088:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761931.639090:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761931.639093:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1365 00000100:00080000:3.0:1789761931.639096:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761931.639100:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789761931.639104:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761931.639113:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872f480 x1876700024636032/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789761931.639143:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789761931.639177:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761931/real 1789761931] req@ffffa06c1872e680 x1876700024635776/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761956 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761931.639191:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872e680 x1876700024635776/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761956 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761931.639199:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761931.639202:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789761931.639205:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024635776:192.168.203.107@tcp:250:kworker.0 00000100:00100000:2.0:1789761931.639235:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024635904:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789761931.639252:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789761931.639262:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761931/real 1789761931] req@ffffa06c1872ea00 x1876700024635904/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761956 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761931.639270:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872ea00 x1876700024635904/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761956 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761931.639277:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761931.639279:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789761931.639281:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024635904:192.168.203.107@tcp:38:kworker.0 00000100:00100000:2.0:1789761931.639293:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024636032:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789761931.639314:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789761931.639322:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761931/real 1789761931] req@ffffa06c1872f480 x1876700024636032/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761956 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761931.639330:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872f480 x1876700024636032/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761956 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761931.639336:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761931.639338:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789761931.639341:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024636032:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789761931.895119:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761932.150956:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761932.918998:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761933.175000:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761933.943073:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761934.198954:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761934.967027:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761935.223053:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761935.991119:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761936.246905:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:3.0:1789761936.758993:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761936.759006:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789761936.759011:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761936.759022:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1370 00000100:00080000:3.0:1789761936.759032:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761936.759055:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:3.0:1789761936.759094:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872fb80 x1876700024636160/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761936.759140:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761936.759158:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761936.759160:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761936.759165:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1370 00000100:00080000:3.0:1789761936.759169:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761936.759174:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789761936.759213:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761936.759222:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024636160:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789761936.759234:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872f480 x1876700024636288/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761936.759251:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761936.759256:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761936.759259:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761936.759264:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1370 00000100:00080000:3.0:1789761936.759280:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761936.759285:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789761936.759290:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1789761936.759295:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789761936.759307:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872ea00 x1876700024636416/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789761936.759359:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761936/real 1789761936] req@ffffa06c1872fb80 x1876700024636160/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761961 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761936.759383:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872fb80 x1876700024636160/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761961 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761936.759392:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761936.759396:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789761936.759400:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024636160:192.168.203.107@tcp:250:kworker.0 00000100:00100000:2.0:1789761936.759421:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024636288:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789761936.759442:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789761936.759453:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761936/real 1789761936] req@ffffa06c1872f480 x1876700024636288/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761961 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761936.759471:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872f480 x1876700024636288/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761961 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761936.759478:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761936.759481:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789761936.759483:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024636288:192.168.203.107@tcp:38:kworker.0 00000100:00100000:2.0:1789761936.759488:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024636416:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789761936.759503:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789761936.759511:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761936/real 1789761936] req@ffffa06c1872ea00 x1876700024636416/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761961 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761936.759520:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872ea00 x1876700024636416/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761961 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761936.759526:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761936.759528:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789761936.759530:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024636416:192.168.203.107@tcp:38:kworker.0 00010000:00010000:1.0:1789761937.014988:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761937.270967:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761938.039017:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761938.294911:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761939.063022:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761939.319001:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761940.087057:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761940.342903:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761941.110955:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761941.366978:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789761941.879022:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761941.879034:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761941.879040:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761941.879053:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1375 00000100:00080000:0.0:1789761941.879066:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761941.879086:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789761941.879120:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0e00 x1876700024636544/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761941.879164:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761941.879168:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761941.879182:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761941.879187:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1375 00000100:00080000:0.0:1789761941.879190:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761941.879194:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789761941.879203:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761941.879215:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d2a00 x1876700024636672/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761941.879225:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761941.879228:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761941.879230:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761941.879233:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1375 00000100:00080000:0.0:1789761941.879236:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761941.879241:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789761941.879244:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024636544:192.168.203.107@tcp:250:kworker.0 00000008:00100000:0.0:1789761941.879246:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761941.879259:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d1500 x1876700024636800/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789761941.879321:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789761941.879368:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761941/real 1789761941] req@ffffa06c091d0e00 x1876700024636544/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761966 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761941.879383:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0e00 x1876700024636544/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761966 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761941.879414:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761941.879418:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789761941.879421:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024636544:192.168.203.107@tcp:250:kworker.0 00000100:00100000:2.0:1789761941.879427:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024636672:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789761941.879445:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789761941.879455:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761941/real 1789761941] req@ffffa06c091d2a00 x1876700024636672/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761966 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761941.879464:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d2a00 x1876700024636672/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761966 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761941.879470:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761941.879473:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789761941.879475:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024636672:192.168.203.107@tcp:38:kworker.0 00000100:00100000:2.0:1789761941.879494:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024636800:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789761941.879506:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789761941.879513:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761941/real 1789761941] req@ffffa06c091d1500 x1876700024636800/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761966 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761941.879522:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d1500 x1876700024636800/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761966 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761941.879528:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761941.879530:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789761941.879532:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024636800:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789761942.135027:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761942.390935:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761943.159018:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761943.414971:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761944.183026:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761944.439030:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761945.206989:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761945.463023:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761946.230958:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761946.486966:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789761946.998944:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761946.998952:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761946.998955:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761946.998963:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1380 00000100:00080000:0.0:1789761946.998972:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761946.998978:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789761946.999009:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d3100 x1876700024636928/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761946.999059:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761946.999062:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761946.999066:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761946.999070:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1380 00000100:00080000:0.0:1789761946.999073:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761946.999076:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789761946.999084:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761946.999094:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d2680 x1876700024637056/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761946.999117:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1789761946.999123:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024636928:192.168.203.107@tcp:250:kworker.0 00000100:00080000:0.0:1789761946.999124:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761946.999126:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761946.999133:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1380 00000100:00080000:0.0:1789761946.999137:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761946.999140:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789761946.999143:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761946.999153:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0380 x1876700024637184/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789761946.999196:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789761946.999238:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761946/real 1789761946] req@ffffa06c091d3100 x1876700024636928/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761971 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761946.999256:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d3100 x1876700024636928/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761971 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761946.999267:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761946.999271:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789761946.999276:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024636928:192.168.203.107@tcp:250:kworker.0 00000100:00100000:2.0:1789761946.999301:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024637056:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789761946.999325:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789761946.999339:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761946/real 1789761946] req@ffffa06c091d2680 x1876700024637056/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761971 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761946.999352:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d2680 x1876700024637056/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761971 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761946.999362:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761946.999366:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789761946.999370:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024637056:192.168.203.107@tcp:38:kworker.0 00000100:00100000:2.0:1789761946.999376:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024637184:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789761946.999397:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789761946.999428:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761946/real 1789761946] req@ffffa06c091d0380 x1876700024637184/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761971 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761946.999447:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0380 x1876700024637184/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761971 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761946.999457:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761946.999461:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789761946.999465:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024637184:192.168.203.107@tcp:38:kworker.0 00010000:00010000:1.0:1789761947.254967:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761947.510935:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761948.278996:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761948.534943:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761949.303003:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761949.558970:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761950.326967:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761950.582958:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761951.350954:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761951.606933:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789761952.119892:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761952.119902:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761952.119906:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761952.119916:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1385 00000100:00080000:0.0:1789761952.119939:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761952.119947:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789761952.119978:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d1f80 x1876700024637312/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761952.120007:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761952.120011:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761952.120012:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761952.120016:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1385 00000100:00080000:0.0:1789761952.120020:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761952.120024:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789761952.120033:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761952.120046:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0a80 x1876700024637440/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789761952.120056:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024637312:192.168.203.107@tcp:250:kworker.0 00000100:00080000:0.0:1789761952.120062:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761952.120078:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761952.120081:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761952.120086:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1385 00000100:00080000:0.0:1789761952.120089:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761952.120093:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789761952.120099:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761952.120111:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0700 x1876700024637568/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789761952.120160:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024637440:192.168.203.107@tcp:38:kworker.0 00000100:00100000:2.0:1789761952.120182:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024637568:192.168.203.107@tcp:38:kworker.0 00010000:00010000:2.0:1789761952.374958:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761952.630968:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761953.399005:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761953.654917:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761954.423035:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761954.678973:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000400:00000100:1.0:1789761955.191056:0:2331:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.107:988 00000400:00000100:1.0:1789761955.191098:0:2331:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.107@tcp at host 192.168.203.107 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1789761955.191112:0:2331:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.107@tcp 00000800:00000100:1.0:1789761955.191141:0:2331:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.107@tcp 00000800:00000100:1.0:1789761955.191148:0:2331:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.107@tcp 00000100:00100000:2.0:1789761955.191192:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761952/real 1789761955] req@ffffa06c091d0a80 x1876700024637440/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761977 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761955.191216:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0a80 x1876700024637440/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761977 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761955.191229:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761955.191234:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789761955.191239:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024637440:192.168.203.107@tcp:38:kworker.0 00000100:00100000:2.0:1789761955.191246:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761952/real 1789761955] req@ffffa06c091d0700 x1876700024637568/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761977 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761955.191275:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0700 x1876700024637568/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761977 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761955.191284:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761955.191288:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789761955.191291:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024637568:192.168.203.107@tcp:38:kworker.0 00000100:00100000:2.0:1789761955.191298:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761952/real 1789761955] req@ffffa06c091d1f80 x1876700024637312/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761977 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761955.191311:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d1f80 x1876700024637312/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761977 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761955.191319:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761955.191322:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789761955.191326:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024637312:192.168.203.107@tcp:250:kworker.0 00010000:00010000:0.0:1789761955.446963:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761955.702923:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761956.470987:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761956.726935:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789761957.238922:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761957.238930:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761957.238933:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761957.238942:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1391 00000100:00080000:0.0:1789761957.238951:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761957.238957:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789761957.238987:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d1f80 x1876700024637696/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761957.239009:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761957.239012:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761957.239014:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761957.239016:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1391 00000100:00080000:0.0:1789761957.239019:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761957.239021:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789761957.239028:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761957.239054:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0700 x1876700024637824/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761957.239063:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761957.239066:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761957.239068:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761957.239072:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1391 00000100:00080000:0.0:1789761957.239076:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1789761957.239080:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024637696:192.168.203.107@tcp:250:kworker.0 00000100:00080000:0.0:1789761957.239084:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789761957.239095:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761957.239124:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0a80 x1876700024637952/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789761957.239150:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789761957.239192:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761957/real 1789761957] req@ffffa06c091d1f80 x1876700024637696/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761982 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761957.239213:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d1f80 x1876700024637696/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761982 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761957.239227:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761957.239409:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789761957.239422:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024637696:192.168.203.107@tcp:250:kworker.0 00000100:00100000:2.0:1789761957.239431:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024637824:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789761957.239458:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789761957.239474:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761957/real 1789761957] req@ffffa06c091d0700 x1876700024637824/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761982 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761957.239490:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0700 x1876700024637824/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761982 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761957.239502:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761957.239507:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789761957.239512:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024637824:192.168.203.107@tcp:38:kworker.0 00000100:00100000:2.0:1789761957.239569:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024637952:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789761957.239596:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789761957.239610:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761957/real 1789761957] req@ffffa06c091d0a80 x1876700024637952/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761982 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761957.239624:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0a80 x1876700024637952/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761982 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761957.239633:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761957.239637:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789761957.239641:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024637952:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789761957.495008:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761957.750936:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789761958.518983:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761958.774965:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789761959.542995:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761959.798899:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761960.567038:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761960.822968:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761961.590976:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761961.846899:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789761962.358958:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761962.358967:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761962.358971:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761962.358981:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1396 00000100:00080000:0.0:1789761962.359007:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761962.359016:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789761962.359049:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0380 x1876700024638080/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761962.359077:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761962.359081:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761962.359083:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761962.359088:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1396 00000100:00080000:0.0:1789761962.359092:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761962.359096:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789761962.359105:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761962.359120:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024638080:192.168.203.107@tcp:250:kworker.0 00000100:00100000:0.0:1789761962.359121:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d2680 x1876700024638208/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761962.359134:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761962.359140:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761962.359142:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761962.359154:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1396 00000100:00080000:0.0:1789761962.359158:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761962.359163:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789761962.359168:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761962.359181:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d3100 x1876700024638336/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789761962.359191:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789761962.359227:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761962/real 1789761962] req@ffffa06c091d0380 x1876700024638080/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761987 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761962.359241:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0380 x1876700024638080/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761987 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761962.359248:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761962.359251:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789761962.359254:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024638080:192.168.203.107@tcp:250:kworker.0 00000100:00100000:2.0:1789761962.359270:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024638208:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789761962.359284:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789761962.359292:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761962/real 1789761962] req@ffffa06c091d2680 x1876700024638208/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761987 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761962.359300:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d2680 x1876700024638208/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761987 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761962.359306:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761962.359308:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789761962.359311:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024638208:192.168.203.107@tcp:38:kworker.0 00000100:00100000:2.0:1789761962.359314:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024638336:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789761962.359324:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789761962.359330:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761962/real 1789761962] req@ffffa06c091d3100 x1876700024638336/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761987 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761962.359364:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d3100 x1876700024638336/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761987 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761962.359370:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761962.359372:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789761962.359374:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024638336:192.168.203.107@tcp:38:kworker.0 00010000:00010000:1.0:1789761962.614983:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761962.870944:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761963.639095:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761963.894968:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761964.663030:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761964.918961:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789761965.686953:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761965.942953:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761966.711021:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761966.966977:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:3.0:1789761967.478983:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761967.479020:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789761967.479024:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761967.479035:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1401 00000100:00080000:3.0:1789761967.479046:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761967.479053:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:3.0:1789761967.479092:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872e680 x1876700024638464/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761967.479126:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761967.479130:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761967.479132:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761967.479135:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1401 00000100:00080000:3.0:1789761967.479139:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761967.479142:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789761967.479151:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761967.479165:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872dc00 x1876700024638592/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761967.479178:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761967.479182:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761967.479184:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761967.479188:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1401 00000100:00100000:2.0:1789761967.479220:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024638464:192.168.203.107@tcp:250:kworker.0 00000100:00080000:3.0:1789761967.479230:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761967.479239:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789761967.479257:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761967.479272:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872c700 x1876700024638720/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789761967.479333:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789761967.479375:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761967/real 1789761967] req@ffffa06c1872e680 x1876700024638464/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761992 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761967.479391:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872e680 x1876700024638464/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761992 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761967.479399:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761967.479402:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789761967.479405:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024638464:192.168.203.107@tcp:250:kworker.0 00000100:00100000:2.0:1789761967.479411:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024638592:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789761967.479428:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789761967.479436:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761967/real 1789761967] req@ffffa06c1872dc00 x1876700024638592/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761992 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761967.479444:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872dc00 x1876700024638592/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761992 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761967.479456:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761967.479458:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789761967.479461:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024638592:192.168.203.107@tcp:38:kworker.0 00000100:00100000:2.0:1789761967.479479:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024638720:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789761967.479492:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789761967.479499:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761967/real 1789761967] req@ffffa06c1872c700 x1876700024638720/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761992 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761967.479506:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872c700 x1876700024638720/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761992 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761967.479511:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761967.479513:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789761967.479515:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024638720:192.168.203.107@tcp:38:kworker.0 00010000:00010000:1.0:1789761967.734994:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761967.990963:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761968.758985:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761969.014916:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761969.783007:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761970.038962:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761970.806961:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761971.062903:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761971.830991:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761972.086905:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789761972.598945:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761972.598953:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789761972.598957:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761972.598965:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1406 00000100:00080000:2.0:1789761972.598973:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761972.598989:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789761972.599021:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51180 x1876700024638848/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761972.599044:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761972.599060:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761972.599061:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761972.599065:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1406 00000100:00080000:2.0:1789761972.599069:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761972.599072:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789761972.599092:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761972.599093:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024638848:192.168.203.107@tcp:250:kworker.0 00000100:00100000:2.0:1789761972.599104:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51880 x1876700024638976/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761972.599114:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761972.599117:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761972.599118:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761972.599121:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1406 00000100:00080000:2.0:1789761972.599123:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761972.599125:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789761972.599128:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761972.599136:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e52300 x1876700024639104/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789761972.599164:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761972.599195:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761972/real 1789761972] req@ffffa06c07e51180 x1876700024638848/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761997 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761972.599208:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51180 x1876700024638848/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761997 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761972.599216:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761972.599220:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761972.599222:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024638848:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789761972.599241:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024638976:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761972.599265:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761972.599273:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761972/real 1789761972] req@ffffa06c07e51880 x1876700024638976/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761997 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761972.599282:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51880 x1876700024638976/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761997 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761972.599287:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761972.599289:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761972.599291:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024638976:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789761972.599295:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e52300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024639104:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761972.599306:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761972.599316:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761972/real 1789761972] req@ffffa06c07e52300 x1876700024639104/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761997 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761972.599354:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e52300 x1876700024639104/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761997 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761972.599364:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761972.599368:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761972.599371:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e52300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024639104:192.168.203.107@tcp:38:kworker.0 00010000:00010000:0.0:1789761972.854968:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761973.110927:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761973.878985:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761974.134972:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761974.903020:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761975.158929:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761975.926962:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761976.182907:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761976.950967:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761977.206925:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789761977.718945:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761977.718953:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789761977.718956:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761977.718965:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1411 00000100:00080000:2.0:1789761977.718975:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761977.718980:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789761977.719025:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50000 x1876700024639232/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761977.719058:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761977.719061:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761977.719062:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761977.719066:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1411 00000100:00080000:2.0:1789761977.719069:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761977.719072:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789761977.719079:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761977.719090:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51500 x1876700024639360/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789761977.719095:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024639232:192.168.203.107@tcp:250:kworker.0 00000100:00080000:2.0:1789761977.719105:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761977.719108:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761977.719110:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761977.719113:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1411 00000100:00080000:2.0:1789761977.719116:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761977.719118:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789761977.719122:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761977.719132:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50e00 x1876700024639488/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789761977.719154:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761977.719201:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761977/real 1789761977] req@ffffa06c07e50000 x1876700024639232/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762002 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761977.719222:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50000 x1876700024639232/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762002 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761977.719231:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761977.719234:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761977.719237:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024639232:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789761977.719257:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024639360:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761977.719291:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761977.719301:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761977/real 1789761977] req@ffffa06c07e51500 x1876700024639360/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762002 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761977.719309:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51500 x1876700024639360/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762002 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761977.719316:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761977.719318:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761977.719321:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024639360:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789761977.719327:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024639488:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761977.719346:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761977.719367:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761977/real 1789761977] req@ffffa06c07e50e00 x1876700024639488/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762002 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761977.719381:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50e00 x1876700024639488/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762002 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761977.719387:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761977.719389:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761977.719391:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024639488:192.168.203.107@tcp:38:kworker.0 00010000:00010000:2.0:1789761977.974981:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761978.230953:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761978.999000:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761979.254919:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761980.022973:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761980.278965:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761981.046974:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761981.302923:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761982.070956:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761982.326958:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789761982.838991:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761982.839002:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761982.839006:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761982.839016:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1416 00000100:00080000:0.0:1789761982.839028:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761982.839037:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789761982.839074:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d1500 x1876700024639616/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761982.839103:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761982.839107:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761982.839109:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761982.839114:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1416 00000100:00080000:0.0:1789761982.839119:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761982.839124:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789761982.839134:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761982.839141:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024639616:192.168.203.107@tcp:250:kworker.0 00000100:00100000:0.0:1789761982.839153:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d2a00 x1876700024639744/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761982.839185:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761982.839189:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761982.839192:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761982.839196:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1416 00000100:00080000:0.0:1789761982.839202:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1789761982.839205:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00080000:0.0:1789761982.839210:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789761982.839218:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761982.839233:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0e00 x1876700024639872/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789761982.839254:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761982/real 1789761982] req@ffffa06c091d1500 x1876700024639616/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762007 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761982.839271:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d1500 x1876700024639616/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762007 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761982.839279:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761982.839282:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761982.839285:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024639616:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789761982.839304:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024639744:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761982.839321:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761982.839330:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761982/real 1789761982] req@ffffa06c091d2a00 x1876700024639744/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762007 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761982.839338:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d2a00 x1876700024639744/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762007 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761982.839343:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761982.839344:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761982.839346:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024639744:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789761982.839350:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024639872:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761982.839360:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761982.839365:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761982/real 1789761982] req@ffffa06c091d0e00 x1876700024639872/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762007 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761982.839380:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0e00 x1876700024639872/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762007 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761982.839386:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761982.839388:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761982.839390:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024639872:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789761983.094948:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761983.350921:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761984.118930:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761984.374945:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761985.143019:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761985.398945:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761986.167004:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761986.422946:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761987.190975:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761987.446991:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789761987.958907:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761987.958915:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761987.958918:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761987.958926:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1421 00000100:00080000:0.0:1789761987.958936:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761987.958942:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789761987.958973:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d3b80 x1876700024640000/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761987.958994:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761987.958996:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761987.958998:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761987.959000:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1421 00000100:00080000:0.0:1789761987.959003:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761987.959005:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789761987.959029:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761987.959039:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d2300 x1876700024640128/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761987.959050:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761987.959055:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761987.959058:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1789761987.959061:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024640000:192.168.203.107@tcp:250:kworker.0 00000100:00080000:0.0:1789761987.959062:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1421 00000100:00080000:0.0:1789761987.959069:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761987.959081:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789761987.959085:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761987.959094:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0000 x1876700024640256/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789761987.959164:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761987.959220:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761987/real 1789761987] req@ffffa06c091d3b80 x1876700024640000/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762012 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761987.959242:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d3b80 x1876700024640000/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762012 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761987.959256:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761987.959261:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761987.959266:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024640000:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789761987.959306:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024640128:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761987.959334:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761987.959349:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761987/real 1789761987] req@ffffa06c091d2300 x1876700024640128/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762012 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761987.959376:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d2300 x1876700024640128/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762012 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761987.959386:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761987.959390:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761987.959394:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024640128:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789761987.959401:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024640256:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761987.959428:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761987.959441:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761987/real 1789761987] req@ffffa06c091d0000 x1876700024640256/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762012 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761987.959453:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0000 x1876700024640256/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762012 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761987.959461:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761987.959465:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761987.959469:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024640256:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789761988.214964:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761988.470956:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761989.238992:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761989.494997:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761990.263018:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761990.518992:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761991.286961:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761991.542982:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789761992.310940:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761992.566923:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789761993.078941:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761993.078950:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789761993.078953:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761993.078963:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1426 00000100:00080000:2.0:1789761993.078972:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761993.078979:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789761993.079028:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e53b80 x1876700024640384/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761993.079054:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761993.079057:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761993.079059:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761993.079062:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1426 00000100:00080000:2.0:1789761993.079066:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761993.079069:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789761993.079077:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761993.079091:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50380 x1876700024640512/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789761993.079094:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e53b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024640384:192.168.203.107@tcp:250:kworker.0 00000100:00080000:2.0:1789761993.079106:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761993.079110:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761993.079112:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761993.079132:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1426 00000100:00080000:2.0:1789761993.079136:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761993.079140:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789761993.079144:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761993.079162:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51f80 x1876700024640640/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789761993.079180:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789761993.079223:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761993/real 1789761993] req@ffffa06c07e53b80 x1876700024640384/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762018 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761993.079244:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e53b80 x1876700024640384/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762018 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761993.079256:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761993.079261:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789761993.079266:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e53b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024640384:192.168.203.107@tcp:250:kworker.0 00000100:00100000:0.0:1789761993.079308:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024640512:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789761993.079342:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789761993.079371:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761993/real 1789761993] req@ffffa06c07e50380 x1876700024640512/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762018 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761993.079385:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50380 x1876700024640512/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762018 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761993.079395:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761993.079399:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789761993.079403:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024640512:192.168.203.107@tcp:38:kworker.0 00000100:00100000:0.0:1789761993.079409:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024640640:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789761993.079428:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789761993.079439:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761993/real 1789761993] req@ffffa06c07e51f80 x1876700024640640/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762018 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761993.079451:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51f80 x1876700024640640/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762018 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761993.079460:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761993.079463:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789761993.079467:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024640640:192.168.203.107@tcp:38:kworker.0 00010000:00010000:2.0:1789761993.334975:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761993.590943:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761994.358933:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761994.614945:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789761995.382973:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761995.638970:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789761996.406985:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761996.662935:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789761997.430955:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761997.686944:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789761998.198954:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761998.198963:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761998.198966:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761998.198974:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1432 00000100:00080000:0.0:1789761998.199002:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761998.199007:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789761998.199038:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0000 x1876700024640768/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761998.199073:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761998.199075:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761998.199077:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761998.199080:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1432 00000100:00080000:0.0:1789761998.199082:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761998.199084:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789761998.199091:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761998.199099:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d2300 x1876700024640896/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761998.199107:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761998.199109:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761998.199111:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761998.199124:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1432 00000100:00080000:0.0:1789761998.199126:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761998.199128:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789761998.199130:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761998.199136:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d3b80 x1876700024641024/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789761998.199164:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024640768:192.168.203.107@tcp:250:kworker.0 00000800:00000100:1.0:1789761998.199217:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761998.199248:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761998/real 1789761998] req@ffffa06c091d0000 x1876700024640768/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762023 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761998.199261:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0000 x1876700024640768/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762023 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761998.199269:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761998.199271:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761998.199275:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024640768:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789761998.199280:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024640896:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761998.199298:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761998.199306:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761998/real 1789761998] req@ffffa06c091d2300 x1876700024640896/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762023 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761998.199315:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d2300 x1876700024640896/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762023 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761998.199320:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761998.199322:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761998.199324:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024640896:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789761998.199328:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024641024:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789761998.199349:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789761998.199366:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761998/real 1789761998] req@ffffa06c091d3b80 x1876700024641024/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762023 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761998.199373:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d3b80 x1876700024641024/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762023 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761998.199379:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761998.199380:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789761998.199382:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024641024:192.168.203.107@tcp:38:kworker.0 00010000:00010000:0.0:1789761998.454978:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761998.710904:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789761999.479073:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761999.734946:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762000.503013:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762000.758924:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762001.527025:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762001.783019:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789762002.551010:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762002.807001:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789762003.318932:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762003.318944:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789762003.318956:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762003.318968:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1437 00000100:00080000:0.0:1789762003.318979:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762003.318986:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789762003.319023:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0e00 x1876700024641152/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762003.319054:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762003.319057:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762003.319059:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762003.319064:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1437 00000100:00080000:0.0:1789762003.319068:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762003.319071:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789762003.319081:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762003.319097:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d2a00 x1876700024641280/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762003.319109:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762003.319113:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762003.319115:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762003.319119:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1437 00000100:00080000:0.0:1789762003.319123:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762003.319127:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789762003.319146:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024641152:192.168.203.107@tcp:250:kworker.0 00000008:00100000:0.0:1789762003.319150:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762003.319173:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d1500 x1876700024641408/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789762003.319270:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762003.319337:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762003/real 1789762003] req@ffffa06c091d0e00 x1876700024641152/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762028 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762003.319360:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0e00 x1876700024641152/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762028 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762003.319374:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762003.319379:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762003.319383:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024641152:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789762003.319392:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024641280:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789762003.319418:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762003.319449:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762003/real 1789762003] req@ffffa06c091d2a00 x1876700024641280/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762028 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762003.319464:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d2a00 x1876700024641280/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762028 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762003.319474:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762003.319478:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762003.319482:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024641280:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789762003.319517:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024641408:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789762003.319540:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762003.319552:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762003/real 1789762003] req@ffffa06c091d1500 x1876700024641408/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762028 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762003.319565:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d1500 x1876700024641408/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762028 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762003.319573:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762003.319577:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762003.319581:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024641408:192.168.203.107@tcp:38:kworker.0 00010000:00010000:2.0:1789762003.575046:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762003.830989:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762004.598975:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762004.855029:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762005.622984:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762005.878983:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762006.647073:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762006.902945:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789762007.671017:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762007.926951:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:3.0:1789762008.438992:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762008.439006:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789762008.439010:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762008.439021:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1442 00000100:00080000:3.0:1789762008.439033:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762008.439040:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:3.0:1789762008.439084:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872ed80 x1876700024641536/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762008.439136:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762008.439139:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762008.439141:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762008.439144:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1442 00000100:00080000:3.0:1789762008.439147:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762008.439150:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789762008.439159:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762008.439169:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872f100 x1876700024641664/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762008.439177:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762008.439181:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762008.439182:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762008.439184:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1442 00000100:00080000:3.0:1789762008.439187:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762008.439189:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789762008.439192:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762008.439202:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872ca80 x1876700024641792/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789762008.439257:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024641536:192.168.203.107@tcp:250:kworker.0 00000800:00000100:1.0:1789762008.439322:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762008.439373:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762008/real 1789762008] req@ffffa06c1872ed80 x1876700024641536/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762033 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762008.439387:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872ed80 x1876700024641536/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762033 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762008.439396:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762008.439400:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762008.439403:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024641536:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789762008.439408:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024641664:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789762008.439424:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762008.439431:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762008/real 1789762008] req@ffffa06c1872f100 x1876700024641664/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762033 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762008.439439:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872f100 x1876700024641664/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762033 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762008.439444:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762008.439446:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762008.439448:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024641664:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789762008.439453:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024641792:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789762008.439471:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762008.439482:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762008/real 1789762008] req@ffffa06c1872ca80 x1876700024641792/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762033 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762008.439494:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872ca80 x1876700024641792/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762033 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762008.439514:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762008.439518:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762008.439522:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024641792:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789762008.694980:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762008.950974:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762009.718994:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762009.974965:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789762010.743012:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762010.998961:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789762011.766980:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762012.022914:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789762012.790980:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762013.046950:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:3.0:1789762013.558951:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762013.558962:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789762013.558965:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762013.558974:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1447 00000100:00080000:3.0:1789762013.559002:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762013.559010:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:3.0:1789762013.559054:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872ce00 x1876700024641920/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762013.559080:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762013.559085:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762013.559087:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762013.559090:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1447 00000100:00080000:3.0:1789762013.559093:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762013.559096:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789762013.559104:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762013.559116:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872d180 x1876700024642048/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789762013.559125:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024641920:192.168.203.107@tcp:250:kworker.0 00000100:00080000:3.0:1789762013.559127:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762013.559135:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762013.559136:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762013.559139:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1447 00000100:00080000:3.0:1789762013.559142:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762013.559144:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789762013.559147:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762013.559163:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872e300 x1876700024642176/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789762013.559193:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762013.559238:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762013/real 1789762013] req@ffffa06c1872ce00 x1876700024641920/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762038 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762013.559257:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872ce00 x1876700024641920/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762038 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762013.559265:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762013.559269:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762013.559272:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024641920:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789762013.559290:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024642048:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789762013.559305:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762013.559313:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762013/real 1789762013] req@ffffa06c1872d180 x1876700024642048/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762038 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762013.559333:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872d180 x1876700024642048/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762038 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762013.559339:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762013.559341:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762013.559343:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024642048:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789762013.559347:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024642176:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789762013.559361:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762013.559367:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762013/real 1789762013] req@ffffa06c1872e300 x1876700024642176/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762038 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762013.559375:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872e300 x1876700024642176/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762038 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762013.559380:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762013.559382:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762013.559384:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024642176:192.168.203.107@tcp:38:kworker.0 00010000:00010000:0.0:1789762013.814988:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762014.070954:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762014.838986:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762015.095047:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789762015.862938:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762016.118923:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789762016.887001:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762017.142998:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789762017.910992:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762018.166928:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:3.0:1789762018.678911:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762018.678918:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789762018.678922:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762018.678930:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1452 00000100:00080000:3.0:1789762018.678939:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762018.678945:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:3.0:1789762018.678980:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872d880 x1876700024642304/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762018.679008:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762018.679012:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762018.679014:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762018.679018:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1452 00000100:00080000:3.0:1789762018.679023:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762018.679027:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789762018.679037:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762018.679044:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024642304:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789762018.679068:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872c380 x1876700024642432/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762018.679080:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762018.679083:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762018.679085:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762018.679088:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1452 00000100:00080000:3.0:1789762018.679091:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762018.679095:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789762018.679099:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762018.679122:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872df80 x1876700024642560/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789762018.679130:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024642432:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789762018.679175:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024642560:192.168.203.107@tcp:38:kworker.0 00010000:00010000:0.0:1789762018.934985:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762019.190985:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789762019.959019:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762020.214926:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762020.982981:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762021.238965:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000400:00000100:2.0:1789762021.751205:0:2328:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.107:988 00000400:00000100:2.0:1789762021.751259:0:2328:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.107@tcp at host 192.168.203.107 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1789762021.751283:0:2328:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.107@tcp 00000800:00000100:2.0:1789762021.751336:0:2328:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.107@tcp 00000800:00000100:2.0:1789762021.751367:0:2328:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.107@tcp 00000100:00100000:1.0:1789762021.751384:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762018/real 1789762021] req@ffffa06c1872c380 x1876700024642432/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762043 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762021.751427:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872c380 x1876700024642432/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762043 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762021.751445:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762021.751453:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762021.751460:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024642432:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789762021.751475:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762018/real 1789762021] req@ffffa06c1872d880 x1876700024642304/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762043 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762021.751493:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872d880 x1876700024642304/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762043 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762021.751503:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762021.751508:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762021.751512:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024642304:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789762021.751542:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762018/real 1789762021] req@ffffa06c1872df80 x1876700024642560/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762043 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762021.751574:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872df80 x1876700024642560/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762043 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762021.751612:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762021.751618:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762021.751622:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024642560:192.168.203.107@tcp:38:kworker.0 00010000:00010000:1.0:1789762022.007059:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762022.262906:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789762023.030987:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762023.286958:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:3.0:1789762023.798946:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762023.798954:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789762023.798958:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762023.798967:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1457 00000100:00080000:3.0:1789762023.798976:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762023.798983:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:3.0:1789762023.799019:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872f800 x1876700024642688/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762023.799049:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762023.799053:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762023.799055:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762023.799059:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1457 00000100:00080000:3.0:1789762023.799065:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762023.799070:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789762023.799080:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762023.799095:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024642688:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789762023.799108:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872df80 x1876700024642816/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762023.799128:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762023.799132:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762023.799134:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762023.799138:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1457 00000100:00080000:3.0:1789762023.799143:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762023.799159:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000800:00000100:1.0:1789762023.799165:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000008:00100000:3.0:1789762023.799169:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762023.799186:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872d880 x1876700024642944/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789762023.799207:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762023/real 1789762023] req@ffffa06c1872f800 x1876700024642688/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762048 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762023.799230:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872f800 x1876700024642688/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762048 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762023.799242:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762023.799247:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762023.799267:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024642688:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789762023.799294:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024642816:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789762023.799332:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762023.799347:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762023/real 1789762023] req@ffffa06c1872df80 x1876700024642816/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762048 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762023.799360:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872df80 x1876700024642816/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762048 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762023.799391:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762023.799396:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762023.799400:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024642816:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789762023.799407:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024642944:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789762023.799432:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762023.799444:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762023/real 1789762023] req@ffffa06c1872d880 x1876700024642944/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762048 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762023.799457:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872d880 x1876700024642944/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762048 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762023.799464:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762023.799468:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762023.799471:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024642944:192.168.203.107@tcp:38:kworker.0 00010000:00010000:2.0:1789762024.054982:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762024.310962:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789762025.078946:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762025.334953:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789762026.102930:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762026.358950:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762027.126981:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762027.382955:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789762028.151013:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762028.406939:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789762028.918936:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762028.918947:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789762028.918952:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762028.918980:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1462 00000100:00080000:2.0:1789762028.918992:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762028.919001:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789762028.919041:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e52680 x1876700024643072/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762028.919066:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762028.919071:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762028.919082:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762028.919087:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1462 00000100:00100000:3.0:1789762028.919089:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e52680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024643072:192.168.203.107@tcp:250:kworker.0 00000100:00080000:2.0:1789762028.919092:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762028.919098:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789762028.919108:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762028.919126:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e53800 x1876700024643200/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762028.919140:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762028.919144:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762028.919147:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762028.919151:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1462 00000800:00000100:3.0:1789762028.919155:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00080000:2.0:1789762028.919156:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762028.919161:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789762028.919166:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762028.919178:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51c00 x1876700024643328/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789762028.919198:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762028/real 1789762028] req@ffffa06c07e52680 x1876700024643072/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762053 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762028.919305:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e52680 x1876700024643072/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762053 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762028.919319:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762028.919324:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789762028.919463:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e52680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024643072:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789762028.919521:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e53800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024643200:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789762028.919551:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789762028.919618:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762028/real 1789762028] req@ffffa06c07e53800 x1876700024643200/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762053 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762028.919636:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e53800 x1876700024643200/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762053 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762028.919646:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762028.919650:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789762028.919655:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e53800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024643200:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789762028.919667:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024643328:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789762028.919688:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789762028.919698:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762028/real 1789762028] req@ffffa06c07e51c00 x1876700024643328/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762053 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762028.919713:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51c00 x1876700024643328/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762053 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762028.919720:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762028.919722:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789762028.919724:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024643328:192.168.203.107@tcp:38:kworker.0 00010000:00010000:1.0:1789762029.175000:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762029.430916:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762030.199054:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762030.454953:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789762031.222949:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762031.478933:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789762032.247085:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762032.502946:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789762033.270980:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762033.526957:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789762034.039001:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762034.039019:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789762034.039023:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762034.039035:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1467 00000100:00080000:2.0:1789762034.039046:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762034.039054:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789762034.039093:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50a80 x1876700024643456/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762034.039145:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762034.039151:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762034.039154:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762034.039162:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1467 00000100:00080000:2.0:1789762034.039168:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762034.039184:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789762034.039195:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762034.039214:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50700 x1876700024643584/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789762034.039214:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024643456:192.168.203.107@tcp:250:kworker.0 00000100:00080000:2.0:1789762034.039237:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762034.039241:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762034.039243:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762034.039247:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1467 00000100:00080000:2.0:1789762034.039251:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762034.039254:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789762034.039258:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762034.039270:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e53480 x1876700024643712/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789762034.039305:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789762034.039359:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762034/real 1789762034] req@ffffa06c07e50a80 x1876700024643456/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762059 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762034.039385:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50a80 x1876700024643456/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762059 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762034.039401:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762034.039407:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789762034.039413:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024643456:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789762034.039441:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024643584:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789762034.039516:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789762034.039537:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762034/real 1789762034] req@ffffa06c07e50700 x1876700024643584/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762059 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762034.039568:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50700 x1876700024643584/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762059 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762034.039581:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762034.039586:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789762034.039592:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024643584:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789762034.039601:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e53480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024643712:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789762034.039633:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789762034.039650:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762034/real 1789762034] req@ffffa06c07e53480 x1876700024643712/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762059 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762034.039668:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e53480 x1876700024643712/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762059 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762034.039680:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762034.039685:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789762034.039689:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e53480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024643712:192.168.203.107@tcp:38:kworker.0 00010000:00010000:0.0:1789762034.294980:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762034.550974:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789762035.318962:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762035.574935:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789762036.342981:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762036.598970:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789762037.367016:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762037.622981:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789762038.391116:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762038.646995:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:1.0:1789762039.158944:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762039.158952:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789762039.158955:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762039.158963:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1472 00000100:00080000:1.0:1789762039.158972:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762039.158977:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789762039.159009:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2d180 x1876700024643840/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762039.159035:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762039.159040:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762039.159042:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762039.159047:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1472 00000100:00080000:1.0:1789762039.159052:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762039.159057:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789762039.159066:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762039.159072:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024643840:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789762039.159085:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2ca80 x1876700024643968/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762039.159096:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762039.159099:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762039.159101:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762039.159114:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1472 00000100:00080000:1.0:1789762039.159119:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762039.159122:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000800:00000100:3.0:1789762039.159126:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000008:00100000:1.0:1789762039.159128:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762039.159141:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2c700 x1876700024644096/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789762039.159166:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762039/real 1789762039] req@ffffa06c08e2d180 x1876700024643840/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762064 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762039.159181:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2d180 x1876700024643840/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762064 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762039.159207:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762039.159212:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789762039.159216:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024643840:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789762039.159246:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024643968:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789762039.159271:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789762039.159282:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762039/real 1789762039] req@ffffa06c08e2ca80 x1876700024643968/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762064 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762039.159290:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2ca80 x1876700024643968/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762064 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762039.159297:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762039.159299:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789762039.159302:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024643968:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789762039.159306:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024644096:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789762039.159318:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789762039.159324:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762039/real 1789762039] req@ffffa06c08e2c700 x1876700024644096/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762064 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762039.159332:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2c700 x1876700024644096/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762064 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762039.159337:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762039.159338:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789762039.159340:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024644096:192.168.203.107@tcp:38:kworker.0 00010000:00010000:0.0:1789762039.414964:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762039.670951:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762040.439066:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762040.694937:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789762041.463053:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762041.718948:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789762042.487022:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762042.742974:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762043.511066:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762043.766943:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789762044.278946:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762044.278955:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789762044.278958:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762044.278967:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1478 00000100:00080000:0.0:1789762044.278977:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762044.278999:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789762044.279030:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d3100 x1876700024644224/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762044.279054:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762044.279057:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762044.279059:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762044.279062:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1478 00000100:00080000:0.0:1789762044.279066:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762044.279068:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789762044.279077:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762044.279089:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d2680 x1876700024644352/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789762044.279103:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024644224:192.168.203.107@tcp:250:kworker.0 00000100:00080000:0.0:1789762044.279106:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762044.279128:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762044.279130:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762044.279135:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1478 00000100:00080000:0.0:1789762044.279139:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762044.279144:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789762044.279149:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762044.279164:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0380 x1876700024644480/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789762044.279178:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789762044.279214:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762044/real 1789762044] req@ffffa06c091d3100 x1876700024644224/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762069 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762044.279227:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d3100 x1876700024644224/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762069 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762044.279235:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762044.279238:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789762044.279240:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024644224:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789762044.279277:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024644352:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789762044.279293:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789762044.279302:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762044/real 1789762044] req@ffffa06c091d2680 x1876700024644352/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762069 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762044.279312:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d2680 x1876700024644352/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762069 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762044.279318:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762044.279321:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789762044.279323:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024644352:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789762044.279327:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024644480:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789762044.279353:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789762044.279361:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762044/real 1789762044] req@ffffa06c091d0380 x1876700024644480/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762069 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762044.279370:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0380 x1876700024644480/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762069 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762044.279376:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762044.279393:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789762044.279397:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024644480:192.168.203.107@tcp:38:kworker.0 00010000:00010000:1.0:1789762044.535131:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762044.790934:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789762045.559009:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762045.815027:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789762046.583025:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762046.838988:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762047.607003:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762047.862990:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789762048.631009:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762048.886963:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789762049.398948:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762049.398955:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789762049.398958:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762049.398966:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1483 00000100:00080000:0.0:1789762049.398975:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762049.398981:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789762049.399012:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0a80 x1876700024644608/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762049.399034:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762049.399037:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762049.399038:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762049.399041:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1483 00000100:00080000:0.0:1789762049.399043:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762049.399045:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789762049.399052:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762049.399060:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0700 x1876700024644736/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762049.399068:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762049.399071:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762049.399073:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1789762049.399076:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024644608:192.168.203.107@tcp:250:kworker.0 00000100:00080000:0.0:1789762049.399077:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1483 00000100:00080000:0.0:1789762049.399083:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762049.399087:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789762049.399104:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762049.399114:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d1f80 x1876700024644864/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789762049.399137:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789762049.399171:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762049/real 1789762049] req@ffffa06c091d0a80 x1876700024644608/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762074 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762049.399185:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0a80 x1876700024644608/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762074 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762049.399193:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762049.399196:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789762049.399199:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024644608:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789762049.399216:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024644736:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789762049.399230:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789762049.399238:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762049/real 1789762049] req@ffffa06c091d0700 x1876700024644736/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762074 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762049.399268:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0700 x1876700024644736/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762074 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762049.399274:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762049.399276:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789762049.399279:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024644736:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789762049.399283:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024644864:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789762049.399298:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789762049.399306:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762049/real 1789762049] req@ffffa06c091d1f80 x1876700024644864/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762074 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762049.399313:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d1f80 x1876700024644864/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762074 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762049.399318:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762049.399320:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789762049.399322:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024644864:192.168.203.107@tcp:38:kworker.0 00010000:00010000:0.0:1789762049.655108:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762049.911002:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789762050.679003:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762050.934916:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789762051.703021:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762051.958971:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789762052.727005:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762052.982945:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789762053.750994:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762054.006975:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789762054.518973:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762054.518989:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789762054.518993:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762054.519005:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1488 00000100:00080000:0.0:1789762054.519017:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762054.519025:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789762054.519070:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d1f80 x1876700024644992/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762054.519098:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762054.519102:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762054.519103:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762054.519106:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1488 00000100:00080000:0.0:1789762054.519109:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762054.519111:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789762054.519121:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762054.519131:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0700 x1876700024645120/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762054.519140:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1789762054.519147:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024644992:192.168.203.107@tcp:250:kworker.0 00000100:00080000:0.0:1789762054.519161:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762054.519163:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762054.519167:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1488 00000100:00080000:0.0:1789762054.519171:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762054.519174:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789762054.519178:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762054.519188:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0a80 x1876700024645248/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789762054.519221:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789762054.519266:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762054/real 1789762054] req@ffffa06c091d1f80 x1876700024644992/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762079 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762054.519281:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d1f80 x1876700024644992/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762079 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762054.519317:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762054.519321:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789762054.519325:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024644992:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789762054.519350:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024645120:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789762054.519374:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789762054.519384:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762054/real 1789762054] req@ffffa06c091d0700 x1876700024645120/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762079 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762054.519393:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0700 x1876700024645120/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762079 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762054.519399:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762054.519402:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789762054.519404:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024645120:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789762054.519408:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024645248:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789762054.519420:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789762054.519426:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762054/real 1789762054] req@ffffa06c091d0a80 x1876700024645248/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762079 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762054.519434:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0a80 x1876700024645248/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762079 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762054.519439:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762054.519441:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789762054.519443:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024645248:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789762054.775021:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762055.030917:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789762055.799036:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762056.054955:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762056.822993:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762057.078917:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762057.847047:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762058.102955:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762058.870988:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762059.126914:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789762059.638961:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762059.638971:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789762059.638975:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762059.638985:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1493 00000100:00080000:2.0:1789762059.639013:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762059.639021:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789762059.639056:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e52a00 x1876700024645376/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762059.639084:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762059.639089:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762059.639091:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762059.639096:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1493 00000100:00080000:2.0:1789762059.639101:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762059.639106:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789762059.639116:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762059.639122:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e52a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024645376:192.168.203.107@tcp:250:kworker.0 00000100:00100000:2.0:1789762059.639136:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e53100 x1876700024645504/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762059.639151:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762059.639156:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762059.639158:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762059.639162:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1493 00000100:00080000:2.0:1789762059.639167:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762059.639171:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000800:00000100:3.0:1789762059.639177:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000008:00100000:2.0:1789762059.639178:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762059.639196:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e52d80 x1876700024645632/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789762059.639216:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762059/real 1789762059] req@ffffa06c07e52a00 x1876700024645376/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762084 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762059.639231:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e52a00 x1876700024645376/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762084 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762059.639239:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762059.639242:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789762059.639245:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e52a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024645376:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789762059.639276:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e53100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024645504:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789762059.639295:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789762059.639304:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762059/real 1789762059] req@ffffa06c07e53100 x1876700024645504/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762084 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762059.639312:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e53100 x1876700024645504/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762084 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762059.639321:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762059.639324:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789762059.639328:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e53100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024645504:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789762059.639346:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e52d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024645632:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789762059.639370:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789762059.639383:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762059/real 1789762059] req@ffffa06c07e52d80 x1876700024645632/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762084 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762059.639392:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e52d80 x1876700024645632/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762084 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762059.639397:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762059.639400:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789762059.639402:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e52d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024645632:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789762059.894974:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762060.150979:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762060.919025:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762061.174937:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762061.943002:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762062.198983:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762062.966990:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762063.223002:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762063.990992:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762064.247000:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:3.0:1789762064.758974:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762064.758980:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789762064.758983:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762064.758992:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1498 00000100:00080000:3.0:1789762064.759002:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762064.759008:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:3.0:1789762064.759037:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872c380 x1876700024645760/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762064.759057:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762064.759060:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762064.759061:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762064.759073:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1498 00000100:00080000:3.0:1789762064.759076:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762064.759080:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789762064.759088:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762064.759089:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024645760:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789762064.759104:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872e300 x1876700024645888/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762064.759115:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762064.759118:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762064.759120:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762064.759123:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1498 00000100:00080000:3.0:1789762064.759127:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762064.759130:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789762064.759134:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762064.759144:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872d180 x1876700024646016/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789762064.759151:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789762064.759194:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762064/real 1789762064] req@ffffa06c1872c380 x1876700024645760/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762089 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762064.759214:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872c380 x1876700024645760/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762089 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762064.759239:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762064.759245:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789762064.759249:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024645760:192.168.203.107@tcp:250:kworker.0 00000100:00100000:0.0:1789762064.759272:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024645888:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789762064.759297:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789762064.759311:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762064/real 1789762064] req@ffffa06c1872e300 x1876700024645888/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762089 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762064.759336:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872e300 x1876700024645888/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762089 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762064.759352:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762064.759356:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789762064.759360:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024645888:192.168.203.107@tcp:38:kworker.0 00000100:00100000:0.0:1789762064.759366:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024646016:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789762064.759390:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789762064.759402:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762064/real 1789762064] req@ffffa06c1872d180 x1876700024646016/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762089 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762064.759414:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872d180 x1876700024646016/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762089 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762064.759423:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762064.759427:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789762064.759430:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024646016:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789762065.014961:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762065.270947:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762066.038967:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762066.294973:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762067.062951:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762067.318967:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762068.086961:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762068.342946:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762069.110955:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762069.366955:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789762069.878956:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762069.878963:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789762069.878966:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762069.878974:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1503 00000100:00080000:2.0:1789762069.878983:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762069.878989:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789762069.879020:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e52d80 x1876700024646144/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762069.879040:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762069.879043:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762069.879045:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762069.879048:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1503 00000100:00080000:2.0:1789762069.879051:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762069.879054:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789762069.879061:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762069.879073:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e53100 x1876700024646272/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762069.879075:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e52d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024646144:192.168.203.107@tcp:250:kworker.0 00000100:00080000:2.0:1789762069.879091:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762069.879093:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762069.879095:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762069.879098:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1503 00000100:00080000:2.0:1789762069.879100:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762069.879103:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789762069.879132:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1789762069.879141:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789762069.879148:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e52a00 x1876700024646400/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762069.879196:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762069/real 1789762069] req@ffffa06c07e52d80 x1876700024646144/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762094 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762069.879213:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e52d80 x1876700024646144/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762094 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762069.879223:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762069.879227:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789762069.879239:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e52d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024646144:192.168.203.107@tcp:250:kworker.0 00000100:00100000:0.0:1789762069.879263:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e53100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024646272:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789762069.879285:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789762069.879295:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762069/real 1789762069] req@ffffa06c07e53100 x1876700024646272/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762094 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762069.879304:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e53100 x1876700024646272/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762094 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762069.879310:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762069.879312:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789762069.879315:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e53100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024646272:192.168.203.107@tcp:38:kworker.0 00000100:00100000:0.0:1789762069.879319:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e52a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024646400:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789762069.879329:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789762069.879335:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762069/real 1789762069] req@ffffa06c07e52a00 x1876700024646400/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762094 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762069.879343:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e52a00 x1876700024646400/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762094 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762069.879348:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762069.879349:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789762069.879351:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e52a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024646400:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789762070.134933:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762070.390942:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762071.158970:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762071.414957:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762072.182959:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762072.438962:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762073.206982:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762073.462947:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762074.230939:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762074.486936:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789762074.999041:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762074.999055:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789762074.999086:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762074.999100:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1508 00000100:00080000:0.0:1789762074.999113:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762074.999122:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789762074.999186:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0380 x1876700024646528/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762074.999220:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762074.999223:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762074.999225:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762074.999229:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1508 00000100:00080000:0.0:1789762074.999234:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762074.999238:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789762074.999251:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762074.999267:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d2680 x1876700024646656/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789762074.999278:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024646528:192.168.203.107@tcp:250:kworker.0 00000100:00080000:0.0:1789762074.999283:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762074.999288:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762074.999292:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762074.999300:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1508 00000100:00080000:0.0:1789762074.999305:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762074.999309:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789762074.999314:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762074.999328:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d3100 x1876700024646784/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789762074.999383:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789762074.999439:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762074/real 1789762074] req@ffffa06c091d0380 x1876700024646528/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762099 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762074.999468:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0380 x1876700024646528/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762099 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762074.999481:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762074.999486:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789762074.999491:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024646528:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789762074.999520:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024646656:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789762074.999547:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789762074.999563:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762074/real 1789762074] req@ffffa06c091d2680 x1876700024646656/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762099 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762074.999577:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d2680 x1876700024646656/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762099 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762074.999587:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762074.999590:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789762074.999594:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024646656:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789762074.999601:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024646784:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789762074.999622:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789762074.999634:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762074/real 1789762074] req@ffffa06c091d3100 x1876700024646784/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762099 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762074.999661:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d3100 x1876700024646784/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762099 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762074.999670:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762074.999673:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789762074.999676:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024646784:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789762075.255007:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762075.510959:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762076.278968:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762076.534923:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762077.302991:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762077.558956:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762078.326965:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762078.582919:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789762079.350989:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762079.607003:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789762080.118920:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762080.118928:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789762080.118931:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762080.118951:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1513 00000100:00080000:0.0:1789762080.118960:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762080.118966:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789762080.118996:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d1500 x1876700024646912/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762080.119017:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762080.119020:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762080.119021:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762080.119023:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1513 00000100:00080000:0.0:1789762080.119026:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762080.119028:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789762080.119035:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762080.119045:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d2a00 x1876700024647040/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762080.119054:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762080.119057:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762080.119058:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762080.119061:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1513 00000100:00080000:0.0:1789762080.119064:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1789762080.119074:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024646912:192.168.203.107@tcp:250:kworker.0 00000100:00080000:0.0:1789762080.119084:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789762080.119099:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762080.119108:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0e00 x1876700024647168/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789762080.119169:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789762080.119209:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762080/real 1789762080] req@ffffa06c091d1500 x1876700024646912/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762105 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762080.119229:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d1500 x1876700024646912/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762105 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762080.119240:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762080.119244:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789762080.119249:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024646912:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789762080.119257:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024647040:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789762080.119277:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789762080.119289:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762080/real 1789762080] req@ffffa06c091d2a00 x1876700024647040/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762105 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762080.119319:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d2a00 x1876700024647040/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762105 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762080.119328:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762080.119332:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789762080.119336:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024647040:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789762080.119359:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024647168:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789762080.119377:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789762080.119388:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762080/real 1789762080] req@ffffa06c091d0e00 x1876700024647168/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762105 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762080.119400:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0e00 x1876700024647168/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762105 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762080.119409:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762080.119412:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789762080.119416:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024647168:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789762080.374998:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762080.630905:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789762081.398978:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762081.654938:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789762082.423076:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762082.678978:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762083.447018:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762083.702948:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789762084.470997:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762084.726928:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:1.0:1789762085.239933:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762085.239947:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789762085.239952:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762085.239964:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1519 00000100:00080000:1.0:1789762085.239999:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762085.240008:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789762085.240046:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2ed80 x1876700024647296/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762085.240069:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762085.240092:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762085.240094:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762085.240101:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1519 00000100:00080000:1.0:1789762085.240105:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762085.240114:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:3.0:1789762085.240115:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024647296:192.168.203.107@tcp:250:kworker.0 00000008:00100000:1.0:1789762085.240127:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762085.240142:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2c000 x1876700024647424/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762085.240152:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762085.240155:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762085.240156:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762085.240159:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1519 00000100:00080000:1.0:1789762085.240163:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762085.240166:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789762085.240169:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762085.240176:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2e300 x1876700024647552/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789762085.240232:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024647424:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789762085.240257:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024647552:192.168.203.107@tcp:38:kworker.0 00010000:00010000:0.0:1789762085.494963:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762085.751059:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762086.518994:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762086.775051:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762087.542951:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762087.798952:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000400:00000100:0.0:1789762088.311081:0:2329:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.107:988 00000400:00000100:0.0:1789762088.311124:0:2329:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.107@tcp at host 192.168.203.107 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1789762088.311141:0:2329:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.107@tcp 00000800:00000100:0.0:1789762088.311180:0:2329:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.107@tcp 00000800:00000100:0.0:1789762088.311192:0:2329:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.107@tcp 00000100:00100000:3.0:1789762088.311226:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762085/real 1789762088] req@ffffa06c08e2c000 x1876700024647424/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762110 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762088.311252:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2c000 x1876700024647424/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762110 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762088.311264:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762088.311270:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789762088.311274:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024647424:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789762088.311283:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762085/real 1789762088] req@ffffa06c08e2e300 x1876700024647552/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762110 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762088.311297:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2e300 x1876700024647552/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762110 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762088.311306:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762088.311310:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789762088.311313:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024647552:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789762088.311321:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762085/real 1789762088] req@ffffa06c08e2ed80 x1876700024647296/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762110 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762088.311333:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2ed80 x1876700024647296/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762110 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762088.311340:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762088.311343:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789762088.311346:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024647296:192.168.203.107@tcp:250:kworker.0 00010000:00010000:0.0:1789762088.566971:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762088.822990:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762089.590939:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762089.846949:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:1.0:1789762090.358937:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762090.358964:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789762090.358969:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762090.358980:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1524 00000100:00080000:1.0:1789762090.358993:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762090.359001:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789762090.359065:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2d500 x1876700024647680/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762090.359088:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762090.359091:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762090.359093:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762090.359096:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1524 00000100:00080000:1.0:1789762090.359108:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762090.359114:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789762090.359125:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762090.359125:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024647680:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789762090.359145:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2ed80 x1876700024647808/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762090.359160:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762090.359164:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762090.359166:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762090.359171:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1524 00000100:00080000:1.0:1789762090.359186:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762090.359191:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789762090.359197:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1789762090.359206:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762090.359214:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2e300 x1876700024647936/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789762090.359251:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762090/real 1789762090] req@ffffa06c08e2d500 x1876700024647680/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762115 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762090.359269:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2d500 x1876700024647680/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762115 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762090.359280:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762090.359284:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789762090.359288:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024647680:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789762090.359311:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024647808:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789762090.359336:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789762090.359346:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762090/real 1789762090] req@ffffa06c08e2ed80 x1876700024647808/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762115 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762090.359358:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2ed80 x1876700024647808/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762115 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762090.359367:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762090.359371:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789762090.359375:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024647808:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789762090.359389:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024647936:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789762090.359412:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789762090.359433:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762090/real 1789762090] req@ffffa06c08e2e300 x1876700024647936/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762115 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762090.359452:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2e300 x1876700024647936/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762115 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762090.359461:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762090.359464:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789762090.359468:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024647936:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789762090.614946:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762090.870962:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789762091.639054:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762091.894923:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762092.662959:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762092.918957:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762093.686928:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762093.942993:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789762094.711059:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762094.966940:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:1.0:1789762095.478944:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762095.478952:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789762095.478956:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762095.478964:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1529 00000100:00080000:1.0:1789762095.478974:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762095.478980:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789762095.479024:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2c000 x1876700024648064/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762095.479046:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762095.479049:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762095.479051:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762095.479054:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1529 00000100:00080000:1.0:1789762095.479056:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762095.479059:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789762095.479066:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762095.479088:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2c700 x1876700024648192/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762095.479098:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1789762095.479100:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024648064:192.168.203.107@tcp:250:kworker.0 00000100:00080000:1.0:1789762095.479101:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762095.479105:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762095.479113:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1529 00000100:00080000:1.0:1789762095.479125:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762095.479128:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789762095.479131:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762095.479141:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2ca80 x1876700024648320/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789762095.479182:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789762095.479224:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762095/real 1789762095] req@ffffa06c08e2c000 x1876700024648064/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762120 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762095.479244:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2c000 x1876700024648064/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762120 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762095.479254:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762095.479258:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789762095.479262:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024648064:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789762095.479284:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024648192:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789762095.479308:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789762095.479332:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762095/real 1789762095] req@ffffa06c08e2c700 x1876700024648192/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762120 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762095.479361:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2c700 x1876700024648192/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762120 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762095.479381:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762095.479385:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789762095.479389:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024648192:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789762095.479395:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024648320:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789762095.479418:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789762095.479431:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762095/real 1789762095] req@ffffa06c08e2ca80 x1876700024648320/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762120 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762095.479444:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2ca80 x1876700024648320/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762120 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762095.479453:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762095.479457:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789762095.479460:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024648320:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789762095.734968:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762095.990900:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762096.758970:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762097.015002:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789762097.783088:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762098.038909:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762098.806994:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762099.063003:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762099.831164:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762100.086984:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789762100.599075:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762100.599086:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789762100.599091:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762100.599101:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1534 00000100:00080000:2.0:1789762100.599113:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762100.599123:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789762100.599165:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e53480 x1876700024648448/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762100.599200:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762100.599205:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762100.599207:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762100.599212:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1534 00000100:00080000:2.0:1789762100.599215:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762100.599220:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789762100.599232:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762100.599263:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e53480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024648448:192.168.203.107@tcp:250:kworker.0 00000100:00100000:2.0:1789762100.599273:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50700 x1876700024648576/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762100.599298:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762100.599305:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762100.599309:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762100.599327:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1534 00000100:00080000:2.0:1789762100.599347:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1789762100.599355:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00080000:2.0:1789762100.599360:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789762100.599372:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762100.599397:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50a80 x1876700024648704/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762100.599421:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762100/real 1789762100] req@ffffa06c07e53480 x1876700024648448/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762125 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762100.599445:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e53480 x1876700024648448/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762125 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762100.599457:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762100.599462:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789762100.599469:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e53480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024648448:192.168.203.107@tcp:250:kworker.0 00000100:00100000:0.0:1789762100.599496:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024648576:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789762100.599544:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789762100.599568:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762100/real 1789762100] req@ffffa06c07e50700 x1876700024648576/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762125 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762100.599594:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50700 x1876700024648576/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762125 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762100.599608:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762100.599612:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789762100.599617:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024648576:192.168.203.107@tcp:38:kworker.0 00000100:00100000:0.0:1789762100.599628:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024648704:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789762100.599662:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789762100.599673:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762100/real 1789762100] req@ffffa06c07e50a80 x1876700024648704/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762125 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762100.599685:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50a80 x1876700024648704/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762125 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762100.599693:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762100.599696:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789762100.599699:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024648704:192.168.203.107@tcp:38:kworker.0 00010000:00010000:1.0:1789762100.855154:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762101.111025:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762101.879057:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762102.134921:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762102.903006:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762103.158952:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762103.927039:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762104.182978:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762104.950975:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762105.206991:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789762105.718938:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762105.718963:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789762105.718966:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762105.718975:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1539 00000100:00080000:2.0:1789762105.718995:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762105.719002:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789762105.719034:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51c00 x1876700024648832/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762105.719067:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762105.719071:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762105.719072:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762105.719075:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1539 00000100:00080000:2.0:1789762105.719078:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762105.719080:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789762105.719088:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762105.719099:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e53800 x1876700024648960/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762105.719115:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024648832:192.168.203.107@tcp:250:kworker.0 00000100:00080000:2.0:1789762105.719134:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762105.719137:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762105.719138:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762105.719142:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1539 00000100:00080000:2.0:1789762105.719146:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762105.719149:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789762105.719153:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762105.719174:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e52680 x1876700024649088/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789762105.719204:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789762105.719261:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762105/real 1789762105] req@ffffa06c07e51c00 x1876700024648832/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762130 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762105.719299:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51c00 x1876700024648832/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762130 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762105.719309:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762105.719314:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789762105.719318:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024648832:192.168.203.107@tcp:250:kworker.0 00000100:00100000:0.0:1789762105.719341:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e53800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024648960:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789762105.719365:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789762105.719377:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762105/real 1789762105] req@ffffa06c07e53800 x1876700024648960/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762130 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762105.719389:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e53800 x1876700024648960/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762130 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762105.719398:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762105.719401:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789762105.719421:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e53800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024648960:192.168.203.107@tcp:38:kworker.0 00000100:00100000:0.0:1789762105.719428:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e52680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024649088:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789762105.719453:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789762105.719466:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762105/real 1789762105] req@ffffa06c07e52680 x1876700024649088/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762130 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762105.719497:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e52680 x1876700024649088/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762130 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762105.719518:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762105.719521:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789762105.719523:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e52680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024649088:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789762105.974978:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762106.231057:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762106.998970:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762107.254954:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762108.023004:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762108.278989:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762109.046964:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762109.303031:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762110.071057:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762110.326984:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789762110.838984:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762110.838992:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789762110.838995:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762110.839004:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1544 00000100:00080000:0.0:1789762110.839013:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762110.839018:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789762110.839050:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d3b80 x1876700024649216/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762110.839073:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762110.839076:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762110.839077:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762110.839080:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1544 00000100:00080000:0.0:1789762110.839082:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762110.839084:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789762110.839091:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762110.839100:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d2300 x1876700024649344/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762110.839110:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762110.839112:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:3.0:1789762110.839118:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024649216:192.168.203.107@tcp:250:kworker.0 00000100:00080000:0.0:1789762110.839129:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762110.839133:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1544 00000100:00080000:0.0:1789762110.839137:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762110.839140:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789762110.839144:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762110.839155:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0000 x1876700024649472/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789762110.839178:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789762110.839222:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762110/real 1789762110] req@ffffa06c091d3b80 x1876700024649216/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762135 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762110.839236:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d3b80 x1876700024649216/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762135 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762110.839243:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762110.839246:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789762110.839249:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024649216:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789762110.839267:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024649344:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789762110.839292:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789762110.839300:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762110/real 1789762110] req@ffffa06c091d2300 x1876700024649344/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762135 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762110.839309:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d2300 x1876700024649344/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762135 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762110.839314:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762110.839316:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789762110.839318:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024649344:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789762110.839322:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024649472:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789762110.839332:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789762110.839338:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762110/real 1789762110] req@ffffa06c091d0000 x1876700024649472/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762135 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762110.839353:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0000 x1876700024649472/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762135 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762110.839368:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762110.839370:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789762110.839372:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024649472:192.168.203.107@tcp:38:kworker.0 00010000:00010000:0.0:1789762111.094982:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762111.350960:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762112.118943:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762112.374983:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789762113.143061:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762113.399028:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789762114.167046:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762114.422963:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762115.191044:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762115.446925:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789762115.958987:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762115.958995:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789762115.958999:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762115.959008:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1549 00000100:00080000:0.0:1789762115.959018:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762115.959024:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789762115.959057:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0000 x1876700024649600/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762115.959078:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762115.959081:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762115.959082:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762115.959085:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1549 00000100:00080000:0.0:1789762115.959088:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762115.959090:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789762115.959097:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762115.959121:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d2300 x1876700024649728/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762115.959130:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762115.959132:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762115.959134:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762115.959139:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1549 00000100:00080000:0.0:1789762115.959143:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1789762115.959147:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024649600:192.168.203.107@tcp:250:kworker.0 00000100:00080000:0.0:1789762115.959152:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789762115.959160:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762115.959174:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d3b80 x1876700024649856/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789762115.959234:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789762115.959289:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762115/real 1789762115] req@ffffa06c091d0000 x1876700024649600/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762140 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762115.959309:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0000 x1876700024649600/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762140 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762115.959321:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762115.959326:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789762115.959330:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024649600:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789762115.959338:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024649728:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789762115.959372:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789762115.959387:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762115/real 1789762115] req@ffffa06c091d2300 x1876700024649728/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762140 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762115.959401:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d2300 x1876700024649728/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762140 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762115.959424:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762115.959436:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789762115.959440:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024649728:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789762115.959477:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024649856:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789762115.959500:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789762115.959514:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762115/real 1789762115] req@ffffa06c091d3b80 x1876700024649856/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762140 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762115.959526:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d3b80 x1876700024649856/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762140 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762115.959535:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762115.959539:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789762115.959543:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024649856:192.168.203.107@tcp:38:kworker.0 00010000:00010000:0.0:1789762116.214983:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762116.470960:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762117.239030:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762117.494914:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762118.262952:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762118.518935:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789762119.287025:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762119.542968:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789762120.310985:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762120.566944:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789762121.078978:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762121.079006:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789762121.079009:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762121.079019:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1554 00000100:00080000:0.0:1789762121.079030:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762121.079036:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789762121.079072:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0e00 x1876700024649984/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762121.079097:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762121.079101:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762121.079102:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762121.079131:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1554 00000100:00080000:0.0:1789762121.079135:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762121.079140:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789762121.079166:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762121.079186:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024649984:192.168.203.107@tcp:250:kworker.0 00000100:00100000:0.0:1789762121.079187:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d2a00 x1876700024650112/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762121.079213:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762121.079217:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762121.079220:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762121.079225:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1554 00000100:00080000:0.0:1789762121.079229:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762121.079234:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789762121.079240:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762121.079258:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d1500 x1876700024650240/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789762121.079276:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789762121.079329:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762121/real 1789762121] req@ffffa06c091d0e00 x1876700024649984/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762146 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762121.079351:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0e00 x1876700024649984/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762146 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762121.079379:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762121.079384:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789762121.079418:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024649984:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789762121.079448:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024650112:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789762121.079495:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789762121.079507:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762121/real 1789762121] req@ffffa06c091d2a00 x1876700024650112/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762146 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762121.079517:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d2a00 x1876700024650112/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762146 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762121.079524:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762121.079527:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789762121.079530:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024650112:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789762121.079534:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024650240:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789762121.079554:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789762121.079561:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762121/real 1789762121] req@ffffa06c091d1500 x1876700024650240/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762146 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762121.079569:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d1500 x1876700024650240/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762146 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762121.079576:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762121.079578:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789762121.079580:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024650240:192.168.203.107@tcp:38:kworker.0 00010000:00010000:2.0:1789762121.335055:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762121.591019:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789762122.359148:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762122.615018:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789762123.383063:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762123.639003:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762124.407046:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762124.662961:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789762125.431038:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762125.686948:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:3.0:1789762126.198924:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762126.198945:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789762126.198961:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762126.198969:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1560 00000100:00080000:3.0:1789762126.198978:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762126.198984:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:3.0:1789762126.199013:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872ce00 x1876700024650368/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762126.199038:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762126.199041:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762126.199042:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762126.199045:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1560 00000100:00080000:3.0:1789762126.199048:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762126.199052:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789762126.199060:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762126.199071:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872ca80 x1876700024650496/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762126.199075:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024650368:192.168.203.107@tcp:250:kworker.0 00000100:00080000:3.0:1789762126.199107:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762126.199112:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762126.199115:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762126.199120:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1560 00000100:00080000:3.0:1789762126.199126:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762126.199131:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789762126.199138:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1789762126.199148:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789762126.199152:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872f100 x1876700024650624/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762126.199186:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762126/real 1789762126] req@ffffa06c1872ce00 x1876700024650368/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762151 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762126.199202:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872ce00 x1876700024650368/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762151 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762126.199211:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762126.199221:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789762126.199224:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024650368:192.168.203.107@tcp:250:kworker.0 00000100:00100000:0.0:1789762126.199244:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024650496:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789762126.199261:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789762126.199269:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762126/real 1789762126] req@ffffa06c1872ca80 x1876700024650496/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762151 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762126.199278:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872ca80 x1876700024650496/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762151 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762126.199284:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762126.199286:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789762126.199289:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024650496:192.168.203.107@tcp:38:kworker.0 00000100:00100000:0.0:1789762126.199292:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024650624:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789762126.199303:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789762126.199309:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762126/real 1789762126] req@ffffa06c1872f100 x1876700024650624/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762151 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762126.199333:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872f100 x1876700024650624/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762151 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762126.199338:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762126.199340:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789762126.199342:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024650624:192.168.203.107@tcp:38:kworker.0 00010000:00010000:1.0:1789762126.455023:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762126.710946:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789762127.479050:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762127.734970:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789762128.503180:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762128.758942:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789762129.527049:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762129.782964:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789762130.550964:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762130.806933:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789762131.318965:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762131.318973:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789762131.318976:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762131.318985:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1565 00000100:00080000:2.0:1789762131.318994:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762131.318999:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789762131.319039:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51f80 x1876700024650752/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762131.319061:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762131.319065:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762131.319080:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762131.319085:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1565 00000100:00080000:2.0:1789762131.319090:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762131.319094:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789762131.319103:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762131.319106:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024650752:192.168.203.107@tcp:250:kworker.0 00000100:00100000:2.0:1789762131.319118:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50380 x1876700024650880/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762131.319130:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762131.319133:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762131.319135:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762131.319138:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1565 00000100:00080000:2.0:1789762131.319141:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762131.319144:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789762131.319149:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762131.319158:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e53b80 x1876700024651008/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789762131.319186:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789762131.319222:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762131/real 1789762131] req@ffffa06c07e51f80 x1876700024650752/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762156 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762131.319235:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51f80 x1876700024650752/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762156 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762131.319244:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762131.319248:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789762131.319252:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024650752:192.168.203.107@tcp:250:kworker.0 00000100:00100000:0.0:1789762131.319275:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024650880:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789762131.319300:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789762131.319318:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762131/real 1789762131] req@ffffa06c07e50380 x1876700024650880/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762156 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762131.319327:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50380 x1876700024650880/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762156 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762131.319333:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762131.319335:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789762131.319337:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024650880:192.168.203.107@tcp:38:kworker.0 00000100:00100000:0.0:1789762131.319341:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e53b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024651008:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789762131.319355:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789762131.319361:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762131/real 1789762131] req@ffffa06c07e53b80 x1876700024651008/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762156 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762131.319379:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e53b80 x1876700024651008/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762156 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762131.319384:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762131.319386:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789762131.319388:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e53b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024651008:192.168.203.107@tcp:38:kworker.0 00010000:00010000:2.0:1789762131.575026:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762131.831049:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789762132.598932:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762132.854918:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789762133.622948:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762133.878951:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789762134.646937:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762134.902917:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789762135.670952:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762135.926971:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789762136.439060:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762136.439071:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789762136.439076:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762136.439087:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1570 00000100:00080000:0.0:1789762136.439101:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762136.439110:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789762136.439150:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d3100 x1876700024651136/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762136.439181:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762136.439185:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762136.439188:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762136.439193:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1570 00000100:00080000:0.0:1789762136.439210:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762136.439216:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789762136.439228:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762136.439234:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024651136:192.168.203.107@tcp:250:kworker.0 00000100:00100000:0.0:1789762136.439273:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d2680 x1876700024651264/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762136.439298:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762136.439305:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762136.439309:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762136.439316:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1570 00000100:00080000:0.0:1789762136.439329:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762136.439335:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000800:00000100:3.0:1789762136.439343:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000008:00100000:0.0:1789762136.439364:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762136.439386:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0380 x1876700024651392/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789762136.439520:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762136/real 1789762136] req@ffffa06c091d3100 x1876700024651136/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762161 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762136.439546:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d3100 x1876700024651136/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762161 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762136.439805:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762136.439813:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789762136.439838:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024651136:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789762136.439985:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024651264:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789762136.440054:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789762136.440084:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762136/real 1789762136] req@ffffa06c091d2680 x1876700024651264/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762161 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762136.440101:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d2680 x1876700024651264/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762161 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762136.440130:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762136.440134:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789762136.440139:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024651264:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789762136.440148:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024651392:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789762136.440178:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789762136.440208:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762136/real 1789762136] req@ffffa06c091d0380 x1876700024651392/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762161 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762136.440223:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0380 x1876700024651392/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762161 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762136.440234:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762136.440252:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789762136.440259:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024651392:192.168.203.107@tcp:38:kworker.0 00010000:00010000:0.0:1789762136.694992:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762136.950960:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789762137.719084:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762137.974940:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762138.742953:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762138.998948:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789762139.766960:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762140.022940:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789762140.791065:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762141.046962:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789762141.558935:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762141.558943:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789762141.558947:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762141.558956:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1575 00000100:00080000:2.0:1789762141.558965:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762141.558971:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789762141.559014:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50e00 x1876700024651520/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762141.559036:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762141.559039:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762141.559040:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762141.559043:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1575 00000100:00080000:2.0:1789762141.559046:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762141.559049:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789762141.559056:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762141.559065:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51500 x1876700024651648/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762141.559073:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762141.559078:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:3.0:1789762141.559078:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024651520:192.168.203.107@tcp:250:kworker.0 00000100:00080000:2.0:1789762141.559081:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762141.559088:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1575 00000100:00080000:2.0:1789762141.559091:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762141.559093:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789762141.559095:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762141.559104:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50000 x1876700024651776/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789762141.559141:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789762141.559183:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762141/real 1789762141] req@ffffa06c07e50e00 x1876700024651520/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762166 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762141.559201:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50e00 x1876700024651520/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762166 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762141.559209:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762141.559212:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789762141.559215:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024651520:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789762141.559232:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024651648:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789762141.559249:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789762141.559258:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762141/real 1789762141] req@ffffa06c07e51500 x1876700024651648/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762166 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762141.559266:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51500 x1876700024651648/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762166 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762141.559282:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762141.559284:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789762141.559287:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024651648:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789762141.559291:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024651776:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789762141.559304:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789762141.559311:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762141/real 1789762141] req@ffffa06c07e50000 x1876700024651776/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762166 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762141.559318:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50000 x1876700024651776/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762166 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762141.559323:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762141.559324:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789762141.559326:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024651776:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789762141.814980:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762142.070930:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789762142.838976:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762143.094948:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762143.862968:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762144.118918:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762144.887053:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762145.142915:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789762145.911002:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762146.166984:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789762146.678986:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762146.679003:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789762146.679008:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762146.679018:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1580 00000100:00080000:0.0:1789762146.679030:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762146.679038:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789762146.679081:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0a80 x1876700024651904/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762146.679116:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762146.679119:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762146.679137:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762146.679141:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1580 00000100:00080000:0.0:1789762146.679145:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762146.679148:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789762146.679159:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762146.679170:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024651904:192.168.203.107@tcp:250:kworker.0 00000100:00100000:0.0:1789762146.679171:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0700 x1876700024652032/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762146.679187:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762146.679190:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762146.679191:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762146.679195:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1580 00000100:00080000:0.0:1789762146.679197:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762146.679199:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789762146.679203:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762146.679211:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d1f80 x1876700024652160/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789762146.679246:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789762146.679298:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762146/real 1789762146] req@ffffa06c091d0a80 x1876700024651904/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762171 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762146.679313:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0a80 x1876700024651904/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762171 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762146.679333:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762146.679336:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789762146.679339:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024651904:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789762146.679362:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024652032:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789762146.679381:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789762146.679390:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762146/real 1789762146] req@ffffa06c091d0700 x1876700024652032/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762171 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762146.679398:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0700 x1876700024652032/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762171 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762146.679403:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762146.679406:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789762146.679408:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024652032:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789762146.679411:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024652160:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789762146.679422:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789762146.679428:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762146/real 1789762146] req@ffffa06c091d1f80 x1876700024652160/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762171 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762146.679436:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d1f80 x1876700024652160/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762171 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762146.679443:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762146.679447:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789762146.679450:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024652160:192.168.203.107@tcp:38:kworker.0 00010000:00010000:2.0:1789762146.934991:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762147.190939:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762147.958942:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762148.214934:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762148.982970:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762149.238933:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762150.006930:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762150.262931:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762151.030976:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762151.286952:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:3.0:1789762151.799010:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762151.799025:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789762151.799031:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762151.799053:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1585 00000100:00080000:3.0:1789762151.799067:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762151.799076:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:3.0:1789762151.799133:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872ed80 x1876700024652288/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762151.799171:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762151.799176:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762151.799178:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762151.799183:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1585 00000100:00080000:3.0:1789762151.799188:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762151.799193:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789762151.799206:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762151.799227:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872c700 x1876700024652416/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762151.799228:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024652288:192.168.203.107@tcp:250:kworker.0 00000100:00080000:3.0:1789762151.799257:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762151.799262:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762151.799265:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762151.799269:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1585 00000100:00080000:3.0:1789762151.799290:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762151.799295:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789762151.799311:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762151.799328:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872dc00 x1876700024652544/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762151.799361:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024652416:192.168.203.107@tcp:38:kworker.0 00000100:00100000:0.0:1789762151.799390:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024652544:192.168.203.107@tcp:38:kworker.0 00010000:00010000:0.0:1789762152.055003:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762152.311010:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762153.078992:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762153.334958:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762154.102985:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762154.358926:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000400:00000100:2.0:1789762154.871074:0:2330:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.107:988 00000400:00000100:2.0:1789762154.871126:0:2330:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.107@tcp at host 192.168.203.107 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1789762154.871142:0:2330:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.107@tcp 00000800:00000100:2.0:1789762154.871183:0:2330:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.107@tcp 00000800:00000100:2.0:1789762154.871192:0:2330:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.107@tcp 00000100:00100000:0.0:1789762154.871231:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762151/real 1789762154] req@ffffa06c1872c700 x1876700024652416/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762176 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762154.871275:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872c700 x1876700024652416/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762176 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762154.871285:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762154.871289:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789762154.871292:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024652416:192.168.203.107@tcp:38:kworker.0 00000100:00100000:0.0:1789762154.871298:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762151/real 1789762154] req@ffffa06c1872dc00 x1876700024652544/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762176 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762154.871322:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872dc00 x1876700024652544/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762176 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762154.871331:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762154.871335:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789762154.871338:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024652544:192.168.203.107@tcp:38:kworker.0 00000100:00100000:0.0:1789762154.871346:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762151/real 1789762154] req@ffffa06c1872ed80 x1876700024652288/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762176 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762154.871358:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872ed80 x1876700024652288/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762176 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762154.871366:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762154.871369:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789762154.871372:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024652288:192.168.203.107@tcp:250:kworker.0 00010000:00010000:1.0:1789762155.126985:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762155.382981:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762156.151016:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762156.406934:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789762156.918988:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762156.918996:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789762156.919000:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762156.919012:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1590 00000100:00080000:2.0:1789762156.919023:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762156.919031:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789762156.919067:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e52300 x1876700024652672/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762156.919093:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762156.919096:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762156.919099:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762156.919103:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1590 00000100:00080000:2.0:1789762156.919107:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762156.919111:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789762156.919122:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762156.919142:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51880 x1876700024652800/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762156.919143:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e52300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024652672:192.168.203.107@tcp:250:kworker.0 00000100:00080000:2.0:1789762156.919157:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762156.919165:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762156.919184:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762156.919190:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1590 00000100:00080000:2.0:1789762156.919195:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762156.919200:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789762156.919207:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762156.919223:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51180 x1876700024652928/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789762156.919224:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789762156.919272:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762156/real 1789762156] req@ffffa06c07e52300 x1876700024652672/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762181 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762156.919292:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e52300 x1876700024652672/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762181 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762156.919303:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762156.919307:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789762156.919311:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e52300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024652672:192.168.203.107@tcp:250:kworker.0 00000100:00100000:0.0:1789762156.919334:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024652800:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789762156.919357:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789762156.919370:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762156/real 1789762156] req@ffffa06c07e51880 x1876700024652800/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762181 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762156.919382:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51880 x1876700024652800/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762181 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762156.919391:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762156.919395:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789762156.919398:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024652800:192.168.203.107@tcp:38:kworker.0 00000100:00100000:0.0:1789762156.919405:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024652928:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789762156.919424:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789762156.919435:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762156/real 1789762156] req@ffffa06c07e51180 x1876700024652928/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762181 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762156.919460:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51180 x1876700024652928/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762181 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762156.919468:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762156.919472:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789762156.919475:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024652928:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789762157.174941:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762157.430956:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762158.199008:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762158.454918:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762159.222969:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762159.478936:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762160.246987:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762160.502935:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762161.270942:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762161.526961:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:3.0:1789762162.038986:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762162.038995:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789762162.038998:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762162.039006:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1595 00000100:00080000:3.0:1789762162.039015:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762162.039021:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:3.0:1789762162.039052:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872e680 x1876700024653056/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762162.039075:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762162.039078:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762162.039079:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762162.039081:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1595 00000100:00080000:3.0:1789762162.039084:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762162.039086:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789762162.039093:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762162.039101:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872ea00 x1876700024653184/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762162.039124:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1789762162.039126:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024653056:192.168.203.107@tcp:250:kworker.0 00000100:00080000:3.0:1789762162.039127:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762162.039131:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762162.039136:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1595 00000100:00080000:3.0:1789762162.039138:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762162.039141:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789762162.039145:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762162.039154:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872f480 x1876700024653312/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789762162.039185:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789762162.039228:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762162/real 1789762162] req@ffffa06c1872e680 x1876700024653056/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762187 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762162.039241:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872e680 x1876700024653056/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762187 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762162.039248:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762162.039252:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789762162.039255:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024653056:192.168.203.107@tcp:250:kworker.0 00000100:00100000:0.0:1789762162.039272:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024653184:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789762162.039286:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789762162.039293:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762162/real 1789762162] req@ffffa06c1872ea00 x1876700024653184/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762187 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762162.039301:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872ea00 x1876700024653184/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762187 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762162.039318:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762162.039320:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789762162.039323:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024653184:192.168.203.107@tcp:38:kworker.0 00000100:00100000:0.0:1789762162.039327:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024653312:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789762162.039340:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789762162.039346:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762162/real 1789762162] req@ffffa06c1872f480 x1876700024653312/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762187 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762162.039354:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872f480 x1876700024653312/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762187 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762162.039358:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762162.039360:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789762162.039362:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024653312:192.168.203.107@tcp:38:kworker.0 00010000:00010000:2.0:1789762162.294968:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762162.550949:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762163.318953:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762163.574932:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789762164.343176:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762164.598952:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762165.366954:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762165.623028:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762166.391079:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762166.647071:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:1.0:1789762167.159008:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762167.159020:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789762167.159025:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762167.159037:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1600 00000100:00080000:1.0:1789762167.159050:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762167.159060:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789762167.159107:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2d180 x1876700024653440/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762167.159143:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762167.159148:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762167.159151:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762167.159156:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1600 00000100:00080000:1.0:1789762167.159161:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762167.159165:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789762167.159176:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762167.159191:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2ca80 x1876700024653568/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762167.159206:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762167.159210:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762167.159235:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762167.159246:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1600 00000100:00080000:1.0:1789762167.159256:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1789762167.159272:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024653440:192.168.203.107@tcp:250:kworker.0 00000100:00080000:1.0:1789762167.159273:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789762167.159294:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762167.159329:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2c700 x1876700024653696/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789762167.159372:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789762167.159425:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762167/real 1789762167] req@ffffa06c08e2d180 x1876700024653440/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762192 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762167.159449:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2d180 x1876700024653440/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762192 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762167.159487:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762167.159493:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789762167.159511:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024653440:192.168.203.107@tcp:250:kworker.0 00000100:00100000:0.0:1789762167.159522:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024653568:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789762167.159555:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789762167.159572:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762167/real 1789762167] req@ffffa06c08e2ca80 x1876700024653568/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762192 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762167.159588:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2ca80 x1876700024653568/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762192 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762167.159598:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762167.159602:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789762167.159607:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024653568:192.168.203.107@tcp:38:kworker.0 00000100:00100000:0.0:1789762167.159637:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024653696:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789762167.159657:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789762167.159668:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762167/real 1789762167] req@ffffa06c08e2c700 x1876700024653696/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762192 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762167.159682:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2c700 x1876700024653696/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762192 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762167.159692:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762167.159695:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789762167.159699:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024653696:192.168.203.107@tcp:38:kworker.0 00010000:00010000:0.0:1789762167.415089:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762167.670964:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762168.438980:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762168.694955:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762169.463008:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762169.718963:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789762170.486994:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762170.742962:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762171.511001:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762171.766957:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:1.0:1789762172.278914:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762172.278923:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789762172.278926:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762172.278934:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1606 00000100:00080000:1.0:1789762172.278943:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762172.278949:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789762172.278999:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2c000 x1876700024653824/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762172.279023:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762172.279025:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762172.279027:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762172.279030:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1606 00000100:00080000:1.0:1789762172.279033:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762172.279047:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789762172.279066:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024653824:192.168.203.107@tcp:250:kworker.0 00000008:00100000:1.0:1789762172.279067:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762172.279090:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2e300 x1876700024653952/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762172.279107:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762172.279112:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762172.279114:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762172.279119:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1606 00000100:00080000:1.0:1789762172.279124:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762172.279131:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000800:00000100:2.0:1789762172.279133:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000008:00100000:1.0:1789762172.279138:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762172.279150:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2ed80 x1876700024654080/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789762172.279171:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762172/real 1789762172] req@ffffa06c08e2c000 x1876700024653824/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762197 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762172.279186:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2c000 x1876700024653824/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762197 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762172.279195:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762172.279197:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789762172.279200:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024653824:192.168.203.107@tcp:250:kworker.0 00000100:00100000:2.0:1789762172.279217:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024653952:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789762172.279243:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789762172.279253:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762172/real 1789762172] req@ffffa06c08e2e300 x1876700024653952/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762197 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762172.279263:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2e300 x1876700024653952/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762197 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762172.279269:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762172.279271:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789762172.279273:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024653952:192.168.203.107@tcp:38:kworker.0 00000100:00100000:2.0:1789762172.279278:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024654080:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789762172.279289:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789762172.279296:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762172/real 1789762172] req@ffffa06c08e2ed80 x1876700024654080/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762197 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762172.279303:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2ed80 x1876700024654080/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762197 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762172.279308:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762172.279309:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789762172.279311:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024654080:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789762172.534965:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762172.790927:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789762173.559067:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762173.814944:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789762174.583174:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762174.839011:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762175.607021:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762175.862962:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789762176.630961:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762176.886944:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789762177.398975:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762177.398982:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789762177.398985:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762177.398993:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1611 00000100:00080000:2.0:1789762177.399001:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762177.399023:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789762177.399053:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51180 x1876700024654208/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762177.399077:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762177.399080:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762177.399081:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762177.399084:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1611 00000100:00080000:2.0:1789762177.399087:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762177.399090:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789762177.399097:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762177.399107:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024654208:192.168.203.107@tcp:250:kworker.0 00000100:00100000:2.0:1789762177.399109:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51880 x1876700024654336/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762177.399120:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762177.399123:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762177.399125:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762177.399128:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1611 00000100:00080000:2.0:1789762177.399130:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762177.399132:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789762177.399135:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762177.399142:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e52300 x1876700024654464/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789762177.399171:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789762177.399203:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762177/real 1789762177] req@ffffa06c07e51180 x1876700024654208/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762202 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762177.399216:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51180 x1876700024654208/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762202 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762177.399225:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762177.399244:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789762177.399249:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024654208:192.168.203.107@tcp:250:kworker.0 00000100:00100000:0.0:1789762177.399279:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024654336:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789762177.399308:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789762177.399338:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762177/real 1789762177] req@ffffa06c07e51880 x1876700024654336/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762202 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762177.399354:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51880 x1876700024654336/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762202 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762177.399365:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762177.399369:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789762177.399373:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024654336:192.168.203.107@tcp:38:kworker.0 00000100:00100000:0.0:1789762177.399380:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e52300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024654464:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789762177.399440:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789762177.399448:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762177/real 1789762177] req@ffffa06c07e52300 x1876700024654464/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762202 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762177.399466:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e52300 x1876700024654464/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762202 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762177.399472:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762177.399474:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789762177.399476:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e52300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024654464:192.168.203.107@tcp:38:kworker.0 00010000:00010000:1.0:1789762177.654981:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762177.910987:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762178.678965:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762178.934984:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762179.702973:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762179.958965:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789762180.726984:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762180.983010:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762181.751023:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762182.006991:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:1.0:1789762182.518984:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762182.518995:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789762182.519000:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762182.519011:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1616 00000100:00080000:1.0:1789762182.519022:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762182.519029:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789762182.519072:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2d500 x1876700024654592/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762182.519111:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762182.519117:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762182.519121:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762182.519127:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1616 00000100:00080000:1.0:1789762182.519134:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762182.519142:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789762182.519156:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762182.519180:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024654592:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789762182.519183:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2ed80 x1876700024654720/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762182.519211:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762182.519215:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762182.519217:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762182.519221:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1616 00000100:00080000:1.0:1789762182.519226:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762182.519231:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789762182.519255:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762182.519268:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2e300 x1876700024654848/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789762182.519391:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789762182.519465:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762182/real 1789762182] req@ffffa06c08e2d500 x1876700024654592/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762207 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762182.519502:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2d500 x1876700024654592/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762207 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762182.519523:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762182.519532:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789762182.519540:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024654592:192.168.203.107@tcp:250:kworker.0 00000100:00100000:0.0:1789762182.519588:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024654720:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789762182.519636:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789762182.519665:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762182/real 1789762182] req@ffffa06c08e2ed80 x1876700024654720/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762207 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762182.519693:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2ed80 x1876700024654720/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762207 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762182.519738:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762182.519745:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789762182.519753:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024654720:192.168.203.107@tcp:38:kworker.0 00000100:00100000:0.0:1789762182.519767:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024654848:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789762182.519814:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789762182.519839:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762182/real 1789762182] req@ffffa06c08e2e300 x1876700024654848/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762207 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762182.519950:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2e300 x1876700024654848/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762207 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762182.519968:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762182.519975:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789762182.519982:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024654848:192.168.203.107@tcp:38:kworker.0 00010000:00010000:1.0:1789762182.775013:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762183.030952:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762183.799011:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762184.054974:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789762184.822991:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762185.079023:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789762185.847010:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762186.103004:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762186.870952:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762187.126915:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:1.0:1789762187.638950:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762187.638958:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789762187.638962:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762187.638970:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1621 00000100:00080000:1.0:1789762187.638979:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762187.638996:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789762187.639028:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2c000 x1876700024654976/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762187.639053:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762187.639056:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762187.639061:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762187.639064:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1621 00000100:00080000:1.0:1789762187.639078:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762187.639081:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789762187.639088:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762187.639113:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2c700 x1876700024655104/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762187.639122:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024654976:192.168.203.107@tcp:250:kworker.0 00000100:00080000:1.0:1789762187.639124:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762187.639131:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762187.639132:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762187.639136:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1621 00000100:00080000:1.0:1789762187.639138:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762187.639141:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789762187.639155:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762187.639163:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2ca80 x1876700024655232/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789762187.639221:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789762187.639269:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762187/real 1789762187] req@ffffa06c08e2c000 x1876700024654976/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762212 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762187.639291:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2c000 x1876700024654976/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762212 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762187.639303:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762187.639346:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789762187.639366:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024654976:192.168.203.107@tcp:250:kworker.0 00000100:00100000:0.0:1789762187.639386:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024655104:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789762187.639426:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789762187.639451:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762187/real 1789762187] req@ffffa06c08e2c700 x1876700024655104/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762212 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762187.639479:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2c700 x1876700024655104/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762212 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762187.639484:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762187.639487:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789762187.639489:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024655104:192.168.203.107@tcp:38:kworker.0 00000100:00100000:0.0:1789762187.639492:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024655232:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789762187.639503:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789762187.639509:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762187/real 1789762187] req@ffffa06c08e2ca80 x1876700024655232/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762212 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762187.639516:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2ca80 x1876700024655232/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762212 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762187.639521:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762187.639531:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789762187.639533:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024655232:192.168.203.107@tcp:38:kworker.0 00010000:00010000:1.0:1789762187.894999:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762188.151005:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762188.918986:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762189.174927:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789762189.943044:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762190.198979:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762190.966991:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762191.222927:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762191.990997:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762192.246970:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789762192.758992:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762192.759023:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789762192.759028:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762192.759039:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1626 00000100:00080000:0.0:1789762192.759050:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762192.759057:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789762192.759099:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d1f80 x1876700024655360/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762192.759128:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762192.759131:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762192.759132:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762192.759135:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1626 00000100:00080000:0.0:1789762192.759138:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762192.759141:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789762192.759151:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762192.759161:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0700 x1876700024655488/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762192.759171:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762192.759174:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762192.759175:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762192.759178:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1626 00000100:00080000:0.0:1789762192.759180:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762192.759185:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789762192.759185:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024655360:192.168.203.107@tcp:250:kworker.0 00000008:00100000:0.0:1789762192.759193:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762192.759202:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0a80 x1876700024655616/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789762192.759259:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789762192.759315:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762192/real 1789762192] req@ffffa06c091d1f80 x1876700024655360/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762217 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762192.759336:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d1f80 x1876700024655360/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762217 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762192.759349:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762192.759353:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789762192.759358:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024655360:192.168.203.107@tcp:250:kworker.0 00000100:00100000:2.0:1789762192.759367:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024655488:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789762192.759393:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789762192.759404:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762192/real 1789762192] req@ffffa06c091d0700 x1876700024655488/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762217 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762192.759413:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0700 x1876700024655488/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762217 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762192.759419:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762192.759421:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789762192.759424:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024655488:192.168.203.107@tcp:38:kworker.0 00000100:00100000:2.0:1789762192.759447:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024655616:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789762192.759460:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789762192.759466:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762192/real 1789762192] req@ffffa06c091d0a80 x1876700024655616/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762217 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762192.759482:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0a80 x1876700024655616/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762217 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762192.759487:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762192.759489:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789762192.759500:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024655616:192.168.203.107@tcp:38:kworker.0 00010000:00010000:2.0:1789762193.015030:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762193.270941:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762194.038970:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762194.294946:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762195.062941:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762195.318917:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789762196.087034:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762196.342954:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762197.110974:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762197.366911:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789762197.878997:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762197.879012:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789762197.879017:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762197.879030:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1631 00000100:00080000:0.0:1789762197.879044:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762197.879053:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789762197.879102:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0380 x1876700024655744/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762197.879137:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762197.879141:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762197.879143:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762197.879148:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1631 00000100:00080000:0.0:1789762197.879152:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762197.879156:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789762197.879168:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762197.879182:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d2680 x1876700024655872/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762197.879195:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762197.879199:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762197.879201:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762197.879209:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1631 00000100:00100000:2.0:1789762197.879218:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024655744:192.168.203.107@tcp:250:kworker.0 00000100:00080000:0.0:1789762197.879221:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762197.879270:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789762197.879277:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762197.879291:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d3100 x1876700024656000/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789762197.879320:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789762197.879374:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762197/real 1789762197] req@ffffa06c091d0380 x1876700024655744/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762222 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762197.879397:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0380 x1876700024655744/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762222 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762197.879411:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762197.879416:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789762197.879421:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024655744:192.168.203.107@tcp:250:kworker.0 00000100:00100000:2.0:1789762197.879430:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024655872:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789762197.879455:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789762197.879468:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762197/real 1789762197] req@ffffa06c091d2680 x1876700024655872/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762222 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762197.879481:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d2680 x1876700024655872/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762222 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762197.879511:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762197.879515:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789762197.879519:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024655872:192.168.203.107@tcp:38:kworker.0 00000100:00100000:2.0:1789762197.879544:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024656000:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789762197.879568:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789762197.879580:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762197/real 1789762197] req@ffffa06c091d3100 x1876700024656000/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762222 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762197.879593:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d3100 x1876700024656000/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762222 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762197.879603:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762197.879607:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789762197.879610:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024656000:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789762198.134972:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762198.390955:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762199.159021:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762199.414922:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762200.182964:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762200.438951:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762201.206989:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762201.462992:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762202.230966:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762202.486993:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789762202.999045:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762202.999054:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789762202.999058:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762202.999068:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1636 00000100:00080000:2.0:1789762202.999090:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762202.999097:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789762202.999134:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50000 x1876700024656128/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762202.999183:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762202.999186:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762202.999187:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762202.999202:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1636 00000100:00080000:2.0:1789762202.999206:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762202.999209:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789762202.999217:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762202.999228:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51500 x1876700024656256/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762202.999250:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762202.999253:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762202.999255:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762202.999259:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1636 00000100:00080000:2.0:1789762202.999263:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762202.999267:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789762202.999272:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762202.999284:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50e00 x1876700024656384/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762202.999325:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024656128:192.168.203.107@tcp:250:kworker.0 00000800:00000100:0.0:1789762202.999416:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789762202.999463:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762202/real 1789762202] req@ffffa06c07e50000 x1876700024656128/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762227 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762202.999480:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50000 x1876700024656128/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762227 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762202.999490:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762202.999496:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789762202.999500:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024656128:192.168.203.107@tcp:250:kworker.0 00000100:00100000:0.0:1789762202.999510:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024656256:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789762202.999535:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789762202.999567:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762202/real 1789762202] req@ffffa06c07e51500 x1876700024656256/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762227 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762202.999580:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51500 x1876700024656256/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762227 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762202.999589:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762202.999593:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789762202.999596:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024656256:192.168.203.107@tcp:38:kworker.0 00000100:00100000:0.0:1789762202.999620:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024656384:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789762202.999644:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789762202.999655:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762202/real 1789762202] req@ffffa06c07e50e00 x1876700024656384/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762227 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762202.999667:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50e00 x1876700024656384/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762227 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762202.999676:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762202.999681:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789762202.999686:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024656384:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789762203.255006:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762203.510954:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762204.278978:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762204.534957:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762205.303126:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762205.559013:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762206.326948:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762206.582932:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762207.350988:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762207.606985:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789762208.118982:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762208.118993:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789762208.118997:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762208.119010:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1641 00000100:00080000:0.0:1789762208.119023:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762208.119030:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789762208.119070:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d1500 x1876700024656512/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762208.119099:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762208.119102:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762208.119104:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762208.119107:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1641 00000100:00080000:0.0:1789762208.119111:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762208.119121:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789762208.119132:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762208.119135:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024656512:192.168.203.107@tcp:250:kworker.0 00000100:00100000:0.0:1789762208.119148:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d2a00 x1876700024656640/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762208.119174:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762208.119177:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762208.119180:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762208.119183:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1641 00000100:00080000:0.0:1789762208.119188:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762208.119192:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789762208.119196:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762208.119208:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0e00 x1876700024656768/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789762208.119235:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762208.119313:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762208/real 1789762208] req@ffffa06c091d1500 x1876700024656512/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762233 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762208.119332:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d1500 x1876700024656512/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762233 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762208.119341:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762208.119345:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762208.119348:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024656512:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789762208.119371:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024656640:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789762208.119390:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762208.119399:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762208/real 1789762208] req@ffffa06c091d2a00 x1876700024656640/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762233 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762208.119409:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d2a00 x1876700024656640/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762233 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762208.119415:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762208.119418:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762208.119421:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024656640:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789762208.119424:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024656768:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789762208.119448:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762208.119471:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762208/real 1789762208] req@ffffa06c091d0e00 x1876700024656768/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762233 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762208.119489:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0e00 x1876700024656768/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762233 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762208.119499:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762208.119503:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762208.119507:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024656768:192.168.203.107@tcp:38:kworker.0 00010000:00010000:2.0:1789762208.374951:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762208.630942:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762209.398974:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762209.654983:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762210.423043:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762210.678970:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762211.446981:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762211.702957:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762212.471020:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762212.726925:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789762213.238954:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762213.238962:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789762213.238965:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762213.238974:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1647 00000100:00080000:0.0:1789762213.238982:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762213.238990:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789762213.239021:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d3b80 x1876700024656896/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762213.239046:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762213.239051:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762213.239068:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762213.239074:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1647 00000100:00080000:0.0:1789762213.239079:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762213.239084:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789762213.239089:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024656896:192.168.203.107@tcp:250:kworker.0 00000008:00100000:0.0:1789762213.239093:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762213.239106:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d2300 x1876700024657024/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762213.239131:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762213.239134:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762213.239135:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762213.239139:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1647 00000100:00080000:0.0:1789762213.239148:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762213.239152:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789762213.239156:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1789762213.239158:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789762213.239168:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0000 x1876700024657152/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789762213.239193:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762213/real 1789762213] req@ffffa06c091d3b80 x1876700024656896/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762238 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762213.239207:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d3b80 x1876700024656896/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762238 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762213.239215:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762213.239218:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762213.239221:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024656896:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789762213.239239:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024657024:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789762213.239270:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762213.239283:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762213/real 1789762213] req@ffffa06c091d2300 x1876700024657024/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762238 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762213.239292:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d2300 x1876700024657024/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762238 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762213.239298:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762213.239300:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762213.239303:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024657024:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789762213.239307:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024657152:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789762213.239320:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762213.239327:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762213/real 1789762213] req@ffffa06c091d0000 x1876700024657152/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762238 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762213.239334:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0000 x1876700024657152/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762238 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762213.239339:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762213.239340:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762213.239342:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024657152:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789762213.494946:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762213.750932:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762214.518983:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762214.774924:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762215.542959:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762215.798941:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789762216.566954:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762216.822969:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789762217.590947:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762217.846963:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789762218.358930:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762218.358940:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789762218.358944:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762218.358954:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1652 00000100:00080000:0.0:1789762218.358963:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762218.358978:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789762218.359031:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0000 x1876700024657280/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762218.359060:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762218.359064:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762218.359066:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762218.359070:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1652 00000100:00080000:0.0:1789762218.359089:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762218.359095:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789762218.359105:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762218.359118:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d2300 x1876700024657408/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789762218.359123:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024657280:192.168.203.107@tcp:250:kworker.0 00000100:00080000:0.0:1789762218.359132:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762218.359139:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762218.359141:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762218.359145:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1652 00000100:00080000:0.0:1789762218.359149:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762218.359153:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789762218.359159:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762218.359170:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d3b80 x1876700024657536/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789762218.359227:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024657408:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789762218.359251:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024657536:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789762218.615047:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762218.870956:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762219.638989:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762219.894956:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789762220.663003:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762220.918959:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000400:00000100:3.0:1789762221.431125:0:2331:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.107:988 00000400:00000100:3.0:1789762221.431174:0:2331:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.107@tcp at host 192.168.203.107 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1789762221.431206:0:2331:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.107@tcp 00000800:00000100:3.0:1789762221.431264:0:2331:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.107@tcp 00000800:00000100:3.0:1789762221.431276:0:2331:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.107@tcp 00000100:00100000:1.0:1789762221.431398:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762218/real 1789762221] req@ffffa06c091d2300 x1876700024657408/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762243 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762221.431435:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d2300 x1876700024657408/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762243 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762221.431447:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762221.431452:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762221.431457:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024657408:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789762221.431466:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762218/real 1789762221] req@ffffa06c091d3b80 x1876700024657536/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762243 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762221.431479:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d3b80 x1876700024657536/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762243 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762221.431487:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762221.431491:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762221.431495:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024657536:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789762221.431502:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762218/real 1789762221] req@ffffa06c091d0000 x1876700024657280/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762243 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762221.431539:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0000 x1876700024657280/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762243 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762221.431547:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762221.431550:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762221.431554:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024657280:192.168.203.107@tcp:250:kworker.0 00010000:00010000:2.0:1789762221.686984:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762221.942940:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762222.711015:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762222.966977:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789762223.478951:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762223.478966:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789762223.478971:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762223.478984:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1657 00000100:00080000:2.0:1789762223.478999:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762223.479008:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789762223.479064:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e53b80 x1876700024657664/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762223.479095:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762223.479100:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762223.479102:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762223.479108:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1657 00000100:00080000:2.0:1789762223.479113:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762223.479118:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789762223.479131:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762223.479147:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50380 x1876700024657792/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789762223.479153:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e53b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024657664:192.168.203.107@tcp:250:kworker.0 00000100:00080000:2.0:1789762223.479178:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762223.479185:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762223.479188:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762223.479192:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1657 00000100:00080000:2.0:1789762223.479200:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762223.479206:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789762223.479227:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762223.479242:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51f80 x1876700024657920/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789762223.479255:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762223.479306:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762223/real 1789762223] req@ffffa06c07e53b80 x1876700024657664/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762248 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762223.479327:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e53b80 x1876700024657664/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762248 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762223.479341:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762223.479346:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762223.479350:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e53b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024657664:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789762223.479390:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024657792:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789762223.479419:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762223.479434:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762223/real 1789762223] req@ffffa06c07e50380 x1876700024657792/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762248 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762223.479447:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50380 x1876700024657792/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762248 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762223.479463:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762223.479468:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762223.479471:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024657792:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789762223.479479:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024657920:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789762223.479501:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762223.479520:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762223/real 1789762223] req@ffffa06c07e51f80 x1876700024657920/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762248 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762223.479533:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51f80 x1876700024657920/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762248 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762223.479543:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762223.479546:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762223.479550:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024657920:192.168.203.107@tcp:38:kworker.0 00010000:00010000:0.0:1789762223.734981:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762223.990942:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762224.759045:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762225.014945:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762225.782969:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762226.038912:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789762226.806981:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762227.062911:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762227.830955:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762228.086927:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789762228.598948:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762228.598958:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789762228.598962:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762228.598972:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1662 00000100:00080000:2.0:1789762228.598984:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762228.598993:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789762228.599044:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e52680 x1876700024658048/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762228.599077:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762228.599082:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762228.599084:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762228.599089:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1662 00000100:00080000:2.0:1789762228.599093:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762228.599097:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789762228.599107:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762228.599122:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e53800 x1876700024658176/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789762228.599125:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e52680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024658048:192.168.203.107@tcp:250:kworker.0 00000100:00080000:2.0:1789762228.599144:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762228.599149:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762228.599164:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762228.599170:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1662 00000100:00080000:2.0:1789762228.599175:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762228.599180:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789762228.599186:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1789762228.599192:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789762228.599201:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51c00 x1876700024658304/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789762228.599241:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762228/real 1789762228] req@ffffa06c07e52680 x1876700024658048/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762253 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762228.599278:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e52680 x1876700024658048/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762253 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762228.599290:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762228.599295:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762228.599299:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e52680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024658048:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789762228.599324:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e53800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024658176:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789762228.599349:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762228.599363:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762228/real 1789762228] req@ffffa06c07e53800 x1876700024658176/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762253 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762228.599377:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e53800 x1876700024658176/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762253 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762228.599386:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762228.599390:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762228.599393:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e53800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024658176:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789762228.599400:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024658304:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789762228.599419:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762228.599431:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762228/real 1789762228] req@ffffa06c07e51c00 x1876700024658304/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762253 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762228.599444:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51c00 x1876700024658304/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762253 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762228.599466:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762228.599470:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762228.599474:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024658304:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789762228.854965:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762229.110928:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789762229.878959:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762230.134914:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762230.902978:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762231.158942:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762231.927023:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762232.182930:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762232.951010:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762233.206914:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789762233.719018:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762233.719029:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789762233.719034:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762233.719045:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1667 00000100:00080000:2.0:1789762233.719074:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762233.719083:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789762233.719128:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50a80 x1876700024658432/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762233.719163:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762233.719168:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762233.719170:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762233.719176:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1667 00000100:00080000:2.0:1789762233.719181:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762233.719202:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789762233.719213:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762233.719232:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024658432:192.168.203.107@tcp:250:kworker.0 00000100:00100000:2.0:1789762233.719233:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50700 x1876700024658560/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762233.719262:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762233.719265:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762233.719267:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762233.719271:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1667 00000100:00080000:2.0:1789762233.719276:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762233.719280:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789762233.719285:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762233.719296:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e53480 x1876700024658688/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789762233.719305:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762233.719344:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762233/real 1789762233] req@ffffa06c07e50a80 x1876700024658432/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762258 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762233.719366:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50a80 x1876700024658432/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762258 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762233.719375:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762233.719379:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762233.719382:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024658432:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789762233.719400:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024658560:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789762233.719417:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762233.719426:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762233/real 1789762233] req@ffffa06c07e50700 x1876700024658560/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762258 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762233.719448:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50700 x1876700024658560/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762258 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762233.719455:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762233.719457:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762233.719460:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024658560:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789762233.719464:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e53480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024658688:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789762233.719478:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762233.719485:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762233/real 1789762233] req@ffffa06c07e53480 x1876700024658688/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762258 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762233.719493:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e53480 x1876700024658688/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762258 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762233.719499:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762233.719501:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762233.719503:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e53480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024658688:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789762233.975084:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762234.230914:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762234.999024:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762235.254966:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762236.023006:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762236.278962:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762237.047127:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762237.303125:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762238.071033:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762238.327006:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:3.0:1789762238.838958:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762238.838972:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789762238.838976:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762238.838989:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1672 00000100:00080000:3.0:1789762238.839002:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762238.839011:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:3.0:1789762238.839076:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872fb80 x1876700024658816/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762238.839108:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762238.839112:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762238.839114:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762238.839120:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1672 00000100:00080000:3.0:1789762238.839124:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762238.839129:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789762238.839141:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762238.839158:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024658816:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789762238.839158:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872f480 x1876700024658944/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762238.839176:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762238.839180:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762238.839182:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762238.839187:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1672 00000100:00080000:3.0:1789762238.839191:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762238.839195:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789762238.839201:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762238.839215:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872ea00 x1876700024659072/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789762238.839236:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762238.839296:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762238/real 1789762238] req@ffffa06c1872fb80 x1876700024658816/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762263 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762238.839335:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872fb80 x1876700024658816/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762263 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762238.839347:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762238.839352:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762238.839356:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024658816:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789762238.839386:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024658944:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789762238.839420:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762238.839433:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762238/real 1789762238] req@ffffa06c1872f480 x1876700024658944/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762263 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762238.839442:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872f480 x1876700024658944/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762263 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762238.839449:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762238.839451:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762238.839454:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024658944:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789762238.839458:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024659072:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789762238.839472:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762238.839479:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762238/real 1789762238] req@ffffa06c1872ea00 x1876700024659072/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762263 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762238.839487:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872ea00 x1876700024659072/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762263 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762238.839503:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762238.839512:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762238.839515:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024659072:192.168.203.107@tcp:38:kworker.0 00010000:00010000:2.0:1789762239.094964:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762239.351041:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789762240.119099:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762240.375032:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762241.142988:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762241.398992:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762242.166979:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762242.422968:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762243.191041:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762243.447007:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:1.0:1789762243.959036:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762243.959045:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789762243.959049:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762243.959061:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1677 00000100:00080000:1.0:1789762243.959072:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762243.959083:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789762243.959118:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2d180 x1876700024659200/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762243.959151:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762243.959155:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762243.959158:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762243.959164:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1677 00000100:00080000:1.0:1789762243.959167:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762243.959171:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789762243.959179:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762243.959195:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024659200:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789762243.959229:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2ca80 x1876700024659328/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762243.959250:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762243.959253:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762243.959257:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762243.959260:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1677 00000100:00080000:1.0:1789762243.959266:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762243.959273:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789762243.959279:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1789762243.959290:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762243.959291:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2c700 x1876700024659456/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789762243.959575:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762243/real 1789762243] req@ffffa06c08e2d180 x1876700024659200/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762268 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762243.959656:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2d180 x1876700024659200/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762268 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762243.959673:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762243.959679:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789762243.959686:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024659200:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789762243.959720:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024659328:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789762243.959754:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789762243.959772:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762243/real 1789762243] req@ffffa06c08e2ca80 x1876700024659328/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762268 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762243.959786:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2ca80 x1876700024659328/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762268 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762243.959796:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762243.959800:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789762243.959805:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024659328:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789762243.959812:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024659456:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789762243.959833:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789762243.959879:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762243/real 1789762243] req@ffffa06c08e2c700 x1876700024659456/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762268 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762243.959915:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2c700 x1876700024659456/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762268 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762243.959940:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762243.959945:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789762243.959951:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024659456:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789762244.215059:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762244.471041:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789762245.239000:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762245.494940:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762246.263028:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762246.518961:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762247.287002:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762247.542940:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789762248.311002:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762248.566957:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789762249.078981:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762249.079007:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789762249.079010:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762249.079020:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1682 00000100:00080000:0.0:1789762249.079029:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762249.079035:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789762249.079065:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0e00 x1876700024659584/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762249.079085:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762249.079087:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762249.079089:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762249.079091:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1682 00000100:00080000:0.0:1789762249.079094:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762249.079096:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789762249.079103:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762249.079112:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d2a00 x1876700024659712/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762249.079121:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762249.079124:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762249.079126:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762249.079131:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1682 00000100:00100000:3.0:1789762249.079131:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024659584:192.168.203.107@tcp:250:kworker.0 00000100:00080000:0.0:1789762249.079136:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762249.079141:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789762249.079143:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762249.079151:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d1500 x1876700024659840/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789762249.079191:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789762249.079240:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762249/real 1789762249] req@ffffa06c091d0e00 x1876700024659584/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762274 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762249.079253:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0e00 x1876700024659584/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762274 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762249.079261:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762249.079264:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789762249.079267:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024659584:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789762249.079285:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024659712:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789762249.079299:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789762249.079307:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762249/real 1789762249] req@ffffa06c091d2a00 x1876700024659712/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762274 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762249.079315:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d2a00 x1876700024659712/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762274 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762249.079320:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762249.079331:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789762249.079333:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024659712:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789762249.079337:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024659840:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789762249.079351:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789762249.079359:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762249/real 1789762249] req@ffffa06c091d1500 x1876700024659840/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762274 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762249.079366:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d1500 x1876700024659840/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762274 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762249.079371:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762249.079373:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789762249.079375:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024659840:192.168.203.107@tcp:38:kworker.0 00010000:00010000:0.0:1789762249.334965:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762249.590934:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762250.358973:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762250.614949:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789762251.383004:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762251.638953:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789762252.406983:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762252.663001:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762253.431042:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762253.686901:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789762254.198972:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762254.198987:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789762254.198991:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762254.199003:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1688 00000100:00080000:0.0:1789762254.199017:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762254.199027:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789762254.199078:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d3100 x1876700024659968/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762254.199105:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762254.199109:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762254.199110:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762254.199114:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1688 00000100:00080000:0.0:1789762254.199117:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762254.199120:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789762254.199131:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762254.199143:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d2680 x1876700024660096/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789762254.199149:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024659968:192.168.203.107@tcp:250:kworker.0 00000100:00080000:0.0:1789762254.199163:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762254.199167:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762254.199169:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762254.199173:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1688 00000100:00080000:0.0:1789762254.199200:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762254.199206:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789762254.199213:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1789762254.199226:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789762254.199231:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0380 x1876700024660224/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789762254.199296:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762254/real 1789762254] req@ffffa06c091d3100 x1876700024659968/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762279 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762254.199315:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d3100 x1876700024659968/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762279 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762254.199324:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762254.199327:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789762254.199331:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024659968:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789762254.199356:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024660096:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789762254.199391:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789762254.199402:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762254/real 1789762254] req@ffffa06c091d2680 x1876700024660096/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762279 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762254.199411:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d2680 x1876700024660096/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762279 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762254.199417:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762254.199420:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789762254.199422:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024660096:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789762254.199427:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024660224:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789762254.199457:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789762254.199464:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762254/real 1789762254] req@ffffa06c091d0380 x1876700024660224/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762279 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762254.199474:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0380 x1876700024660224/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762279 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762254.199483:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762254.199487:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789762254.199490:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024660224:192.168.203.107@tcp:38:kworker.0 00010000:00010000:0.0:1789762254.455025:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762254.710999:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762255.479109:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762255.735004:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789762256.503021:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762256.758984:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762257.527023:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762257.782938:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762258.551004:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762258.806934:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:3.0:1789762259.318978:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762259.318987:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789762259.318991:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762259.319017:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1693 00000100:00080000:3.0:1789762259.319026:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762259.319033:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:3.0:1789762259.319067:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872e680 x1876700024660352/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762259.319093:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762259.319096:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762259.319097:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762259.319105:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1693 00000100:00080000:3.0:1789762259.319109:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762259.319122:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789762259.319133:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762259.319148:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872ed80 x1876700024660480/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789762259.319162:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024660352:192.168.203.107@tcp:250:kworker.0 00000100:00080000:3.0:1789762259.319162:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762259.319171:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762259.319174:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762259.319178:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1693 00000100:00080000:3.0:1789762259.319181:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762259.319196:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789762259.319201:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762259.319213:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872dc00 x1876700024660608/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789762259.319233:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762259.319286:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762259/real 1789762259] req@ffffa06c1872e680 x1876700024660352/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762284 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762259.319306:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872e680 x1876700024660352/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762284 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762259.319314:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762259.319317:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762259.319319:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024660352:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789762259.319339:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024660480:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789762259.319358:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762259.319371:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762259/real 1789762259] req@ffffa06c1872ed80 x1876700024660480/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762284 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762259.319384:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872ed80 x1876700024660480/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762284 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762259.319394:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762259.319398:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762259.319401:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024660480:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789762259.319407:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024660608:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789762259.319423:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762259.319430:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762259/real 1789762259] req@ffffa06c1872dc00 x1876700024660608/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762284 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762259.319444:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872dc00 x1876700024660608/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762284 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762259.319450:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762259.319452:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762259.319454:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024660608:192.168.203.107@tcp:38:kworker.0 00010000:00010000:0.0:1789762259.574968:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762259.830992:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789762260.598996:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762260.854958:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762261.622976:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762261.878948:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762262.646957:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762262.902915:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762263.671017:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762263.926973:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789762264.438944:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762264.438953:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789762264.438957:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762264.438966:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1698 00000100:00080000:2.0:1789762264.438976:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762264.438983:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789762264.439019:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e52a00 x1876700024660736/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762264.439087:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762264.439091:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762264.439094:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762264.439115:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1698 00000100:00080000:2.0:1789762264.439118:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762264.439123:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789762264.439132:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762264.439147:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e53100 x1876700024660864/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789762264.439160:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e52a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024660736:192.168.203.107@tcp:250:kworker.0 00000100:00080000:2.0:1789762264.439162:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762264.439182:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762264.439184:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762264.439187:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1698 00000100:00080000:2.0:1789762264.439193:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762264.439196:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789762264.439199:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762264.439207:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e52d80 x1876700024660992/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789762264.439275:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762264.439347:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762264/real 1789762264] req@ffffa06c07e52a00 x1876700024660736/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762289 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762264.439379:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e52a00 x1876700024660736/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762289 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762264.439387:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762264.439390:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762264.439393:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e52a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024660736:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789762264.439412:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e53100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024660864:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789762264.439430:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762264.439445:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762264/real 1789762264] req@ffffa06c07e53100 x1876700024660864/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762289 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762264.439454:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e53100 x1876700024660864/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762289 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762264.439460:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762264.439462:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762264.439464:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e53100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024660864:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789762264.439468:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e52d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024660992:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789762264.439481:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762264.439487:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762264/real 1789762264] req@ffffa06c07e52d80 x1876700024660992/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762289 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762264.439494:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e52d80 x1876700024660992/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762289 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762264.439500:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762264.439501:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762264.439503:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e52d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024660992:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789762264.694979:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762264.950939:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762265.719013:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762265.974981:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762266.742959:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762266.998944:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762267.767037:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762268.022979:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789762268.791015:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762269.046907:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789762269.558930:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762269.558939:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789762269.558942:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762269.558950:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1703 00000100:00080000:2.0:1789762269.558976:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762269.558984:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789762269.559038:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e52d80 x1876700024661120/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762269.559066:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762269.559069:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762269.559070:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762269.559074:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1703 00000100:00080000:2.0:1789762269.559077:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762269.559079:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789762269.559088:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762269.559099:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e53100 x1876700024661248/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762269.559108:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762269.559111:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762269.559113:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762269.559116:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1703 00000100:00080000:2.0:1789762269.559118:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762269.559121:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789762269.559125:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762269.559128:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e52d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024661120:192.168.203.107@tcp:250:kworker.0 00000100:00100000:2.0:1789762269.559135:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e52a00 x1876700024661376/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789762269.559208:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762269.559276:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762269/real 1789762269] req@ffffa06c07e52d80 x1876700024661120/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762294 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762269.559292:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e52d80 x1876700024661120/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762294 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762269.559317:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762269.559321:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762269.559324:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e52d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024661120:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789762269.559330:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e53100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024661248:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789762269.559347:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762269.559356:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762269/real 1789762269] req@ffffa06c07e53100 x1876700024661248/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762294 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762269.559364:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e53100 x1876700024661248/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762294 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762269.559369:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762269.559371:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762269.559373:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e53100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024661248:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789762269.559393:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e52a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024661376:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789762269.559404:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762269.559411:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762269/real 1789762269] req@ffffa06c07e52a00 x1876700024661376/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762294 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762269.559424:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e52a00 x1876700024661376/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762294 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762269.559445:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762269.559448:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762269.559452:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e52a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024661376:192.168.203.107@tcp:38:kworker.0 00010000:00010000:2.0:1789762269.814952:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762270.070957:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762270.839058:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762271.094985:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789762271.862971:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762272.118954:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789762272.887074:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762273.142951:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762273.911038:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762274.166969:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:1.0:1789762274.679051:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762274.679062:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789762274.679066:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762274.679076:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1708 00000100:00080000:1.0:1789762274.679090:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762274.679100:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789762274.679148:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2c000 x1876700024661504/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762274.679206:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762274.679212:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762274.679215:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762274.679223:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1708 00000100:00080000:1.0:1789762274.679230:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762274.679236:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789762274.679249:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762274.679278:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024661504:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789762274.679281:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2e300 x1876700024661632/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762274.679316:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762274.679320:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762274.679323:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762274.679328:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1708 00000100:00080000:1.0:1789762274.679333:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762274.679338:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789762274.679345:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1789762274.679353:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762274.679361:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2ed80 x1876700024661760/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789762274.679409:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762274/real 1789762274] req@ffffa06c08e2c000 x1876700024661504/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762299 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762274.679436:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2c000 x1876700024661504/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762299 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762274.679463:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762274.679468:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789762274.679472:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024661504:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789762274.679499:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024661632:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789762274.679526:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789762274.679550:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762274/real 1789762274] req@ffffa06c08e2e300 x1876700024661632/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762299 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762274.679563:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2e300 x1876700024661632/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762299 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762274.679571:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762274.679574:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789762274.679587:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024661632:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789762274.679594:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024661760:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789762274.679615:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789762274.679626:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762274/real 1789762274] req@ffffa06c08e2ed80 x1876700024661760/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762299 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762274.679638:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2ed80 x1876700024661760/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762299 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762274.679645:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762274.679648:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789762274.679651:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024661760:192.168.203.107@tcp:38:kworker.0 00010000:00010000:2.0:1789762274.935067:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762275.190939:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762275.959046:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762276.214988:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762276.983019:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762277.238971:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762278.006960:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762278.262922:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762279.031017:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762279.287034:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:1.0:1789762279.798936:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762279.798953:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789762279.798958:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762279.798975:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1713 00000100:00080000:1.0:1789762279.799008:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762279.799018:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789762279.799066:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2d500 x1876700024661888/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762279.799100:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762279.799105:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762279.799126:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762279.799133:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1713 00000100:00080000:1.0:1789762279.799138:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762279.799143:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789762279.799155:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762279.799160:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024661888:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789762279.799175:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2ed80 x1876700024662016/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762279.799191:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762279.799195:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762279.799198:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762279.799203:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1713 00000100:00080000:1.0:1789762279.799208:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762279.799213:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789762279.799220:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762279.799238:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2e300 x1876700024662144/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789762279.799252:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789762279.799307:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762279/real 1789762279] req@ffffa06c08e2d500 x1876700024661888/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762304 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762279.799334:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2d500 x1876700024661888/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762304 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762279.799350:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762279.799354:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789762279.799359:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024661888:192.168.203.107@tcp:250:kworker.0 00000100:00100000:2.0:1789762279.799389:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024662016:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789762279.799414:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789762279.799426:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762279/real 1789762279] req@ffffa06c08e2ed80 x1876700024662016/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762304 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762279.799436:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2ed80 x1876700024662016/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762304 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762279.799444:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762279.799447:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789762279.799449:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024662016:192.168.203.107@tcp:38:kworker.0 00000100:00100000:2.0:1789762279.799464:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024662144:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789762279.799484:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789762279.799494:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762279/real 1789762279] req@ffffa06c08e2e300 x1876700024662144/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762304 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762279.799504:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2e300 x1876700024662144/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762304 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762279.799527:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762279.799530:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789762279.799533:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024662144:192.168.203.107@tcp:38:kworker.0 00010000:00010000:2.0:1789762280.055082:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762280.310965:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789762281.079039:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762281.335031:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762282.103024:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762282.358957:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762283.126964:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762283.382941:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789762284.150979:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762284.406953:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789762284.918967:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762284.918978:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789762284.918982:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762284.918994:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1718 00000100:00080000:2.0:1789762284.919006:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762284.919017:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789762284.919076:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e53480 x1876700024662272/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762284.919109:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762284.919113:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762284.919115:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762284.919119:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1718 00000100:00080000:2.0:1789762284.919136:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762284.919142:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:3.0:1789762284.919149:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e53480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024662272:192.168.203.107@tcp:250:kworker.0 00000008:00100000:2.0:1789762284.919155:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762284.919189:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50700 x1876700024662400/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762284.919198:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762284.919201:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762284.919203:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762284.919206:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1718 00000100:00080000:2.0:1789762284.919209:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762284.919212:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789762284.919216:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762284.919223:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50a80 x1876700024662528/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789762284.919282:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024662400:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789762284.919308:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024662528:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789762285.174994:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762285.430978:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762286.199004:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762286.454969:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762287.223056:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762287.478972:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000400:00000100:0.0:1789762287.991126:0:2328:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.107:988 00000400:00000100:0.0:1789762287.991183:0:2328:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.107@tcp at host 192.168.203.107 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1789762287.991205:0:2328:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.107@tcp 00000800:00000100:0.0:1789762287.991261:0:2328:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.107@tcp 00000800:00000100:0.0:1789762287.991274:0:2328:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.107@tcp 00000100:00100000:3.0:1789762287.991315:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762284/real 1789762287] req@ffffa06c07e50700 x1876700024662400/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762309 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762287.991347:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50700 x1876700024662400/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762309 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762287.991362:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762287.991368:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789762287.991387:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024662400:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789762287.991399:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762284/real 1789762287] req@ffffa06c07e50a80 x1876700024662528/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762309 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762287.991421:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50a80 x1876700024662528/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762309 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762287.991431:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762287.991435:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789762287.991439:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024662528:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789762287.991447:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762284/real 1789762287] req@ffffa06c07e53480 x1876700024662272/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762309 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762287.991461:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e53480 x1876700024662272/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762309 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762287.991469:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762287.991473:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789762287.991477:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e53480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024662272:192.168.203.107@tcp:250:kworker.0 00010000:00010000:3.0:1789762288.247000:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762288.502996:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762289.270980:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762289.526942:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:3.0:1789762290.038959:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762290.038969:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789762290.038973:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762290.038984:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1723 00000100:00080000:3.0:1789762290.039011:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762290.039018:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:3.0:1789762290.039070:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872c700 x1876700024662656/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762290.039124:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762290.039146:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762290.039148:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762290.039154:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1723 00000100:00080000:3.0:1789762290.039158:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762290.039163:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789762290.039173:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762290.039190:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872f100 x1876700024662784/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762290.039204:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762290.039208:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762290.039210:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762290.039222:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1723 00000100:00080000:3.0:1789762290.039226:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762290.039230:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789762290.039236:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762290.039247:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872ca80 x1876700024662912/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789762290.039331:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024662656:192.168.203.107@tcp:250:kworker.0 00000800:00000100:2.0:1789762290.039405:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789762290.039449:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762290/real 1789762290] req@ffffa06c1872c700 x1876700024662656/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762315 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762290.039469:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872c700 x1876700024662656/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762315 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762290.039494:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762290.039498:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789762290.039503:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024662656:192.168.203.107@tcp:250:kworker.0 00000100:00100000:2.0:1789762290.039511:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024662784:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789762290.039553:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789762290.039569:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762290/real 1789762290] req@ffffa06c1872f100 x1876700024662784/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762315 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762290.039583:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872f100 x1876700024662784/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762315 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762290.039594:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762290.039598:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789762290.039601:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024662784:192.168.203.107@tcp:38:kworker.0 00000100:00100000:2.0:1789762290.039609:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024662912:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789762290.039633:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789762290.039646:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762290/real 1789762290] req@ffffa06c1872ca80 x1876700024662912/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762315 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762290.039660:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872ca80 x1876700024662912/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762315 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762290.039668:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762290.039672:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789762290.039676:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024662912:192.168.203.107@tcp:38:kworker.0 00010000:00010000:2.0:1789762290.295039:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762290.550918:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762291.318936:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762291.574940:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789762292.343074:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762292.598923:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789762293.367051:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762293.622999:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762294.391004:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762294.646985:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:3.0:1789762295.158932:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762295.158958:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789762295.158962:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762295.158970:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1728 00000100:00080000:3.0:1789762295.158980:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762295.158988:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:3.0:1789762295.159025:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872ce00 x1876700024663040/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762295.159058:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762295.159061:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762295.159062:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762295.159065:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1728 00000100:00080000:3.0:1789762295.159068:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762295.159072:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789762295.159080:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762295.159091:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024663040:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789762295.159094:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872d180 x1876700024663168/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762295.159120:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762295.159124:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762295.159126:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762295.159130:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1728 00000100:00080000:3.0:1789762295.159135:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762295.159140:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789762295.159145:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762295.159158:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872e300 x1876700024663296/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789762295.159163:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789762295.159202:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762295/real 1789762295] req@ffffa06c1872ce00 x1876700024663040/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762320 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762295.159215:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872ce00 x1876700024663040/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762320 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762295.159224:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762295.159226:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789762295.159230:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024663040:192.168.203.107@tcp:250:kworker.0 00000100:00100000:2.0:1789762295.159248:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024663168:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789762295.159262:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789762295.159269:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762295/real 1789762295] req@ffffa06c1872d180 x1876700024663168/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762320 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762295.159277:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872d180 x1876700024663168/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762320 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762295.159283:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762295.159285:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789762295.159287:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024663168:192.168.203.107@tcp:38:kworker.0 00000100:00100000:2.0:1789762295.159290:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024663296:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789762295.159300:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789762295.159316:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762295/real 1789762295] req@ffffa06c1872e300 x1876700024663296/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762320 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762295.159323:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872e300 x1876700024663296/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762320 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762295.159329:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762295.159331:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789762295.159333:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024663296:192.168.203.107@tcp:38:kworker.0 00010000:00010000:1.0:1789762295.414949:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762295.670925:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762296.438967:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762296.694953:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762297.462952:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762297.718943:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789762298.487004:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762298.742985:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762299.511101:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762299.766979:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:3.0:1789762300.278935:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762300.278944:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789762300.278948:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762300.278956:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1734 00000100:00080000:3.0:1789762300.278973:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762300.278980:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:3.0:1789762300.279019:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872c380 x1876700024663424/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762300.279065:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762300.279070:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762300.279072:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762300.279078:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1734 00000100:00080000:3.0:1789762300.279083:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762300.279089:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789762300.279100:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762300.279129:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872d880 x1876700024663552/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789762300.279140:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024663424:192.168.203.107@tcp:250:kworker.0 00000100:00080000:3.0:1789762300.279141:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762300.279150:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762300.279152:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762300.279156:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1734 00000100:00080000:3.0:1789762300.279161:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762300.279181:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789762300.279188:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1789762300.279198:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789762300.279203:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872df80 x1876700024663680/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789762300.279236:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762300/real 1789762300] req@ffffa06c1872c380 x1876700024663424/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762325 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762300.279251:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872c380 x1876700024663424/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762325 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762300.279258:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762300.279261:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789762300.279264:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024663424:192.168.203.107@tcp:250:kworker.0 00000100:00100000:2.0:1789762300.279281:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024663552:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789762300.279296:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789762300.279304:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762300/real 1789762300] req@ffffa06c1872d880 x1876700024663552/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762325 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762300.279324:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872d880 x1876700024663552/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762325 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762300.279331:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762300.279333:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789762300.279335:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024663552:192.168.203.107@tcp:38:kworker.0 00000100:00100000:2.0:1789762300.279339:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024663680:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789762300.279352:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789762300.279359:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762300/real 1789762300] req@ffffa06c1872df80 x1876700024663680/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762325 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762300.279366:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872df80 x1876700024663680/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762325 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762300.279371:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762300.279373:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789762300.279382:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024663680:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789762300.534997:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762300.790953:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789762301.559039:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762301.814937:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789762302.583034:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762302.838960:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762303.606953:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762303.862921:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762304.631022:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762304.887039:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789762305.399003:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762305.399017:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789762305.399022:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762305.399035:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1739 00000100:00080000:2.0:1789762305.399049:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762305.399059:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789762305.399107:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51c00 x1876700024663808/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762305.399156:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762305.399161:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762305.399163:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762305.399168:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1739 00000100:00080000:2.0:1789762305.399184:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762305.399190:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789762305.399200:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762305.399211:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024663808:192.168.203.107@tcp:250:kworker.0 00000100:00100000:2.0:1789762305.399221:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e53800 x1876700024663936/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762305.399243:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762305.399249:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762305.399252:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762305.399258:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1739 00000100:00080000:2.0:1789762305.399276:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1789762305.399287:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00080000:2.0:1789762305.399288:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789762305.399310:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762305.399328:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e52680 x1876700024664064/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789762305.399353:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762305/real 1789762305] req@ffffa06c07e51c00 x1876700024663808/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762330 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762305.399373:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51c00 x1876700024663808/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762330 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762305.399384:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762305.399388:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789762305.399392:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024663808:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789762305.399418:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e53800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024663936:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789762305.399468:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789762305.399506:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762305/real 1789762305] req@ffffa06c07e53800 x1876700024663936/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762330 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762305.399521:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e53800 x1876700024663936/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762330 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762305.399530:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762305.399545:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789762305.399548:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e53800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024663936:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789762305.399555:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e52680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024664064:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789762305.399580:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789762305.399593:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762305/real 1789762305] req@ffffa06c07e52680 x1876700024664064/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762330 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762305.399607:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e52680 x1876700024664064/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762330 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762305.399616:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762305.399621:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789762305.399624:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e52680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024664064:192.168.203.107@tcp:38:kworker.0 00010000:00010000:0.0:1789762305.654947:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762305.910930:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762306.678969:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762306.935014:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762307.702995:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762307.958931:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789762308.727090:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762308.982984:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762309.751012:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762310.006922:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789762310.518957:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762310.518968:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789762310.518972:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762310.518984:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1744 00000100:00080000:0.0:1789762310.518996:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762310.519028:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789762310.519067:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0a80 x1876700024664192/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762310.519094:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762310.519098:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762310.519100:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762310.519105:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1744 00000100:00080000:0.0:1789762310.519110:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762310.519113:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789762310.519125:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762310.519139:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0700 x1876700024664320/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789762310.519148:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024664192:192.168.203.107@tcp:250:kworker.0 00000100:00080000:0.0:1789762310.519152:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762310.519155:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762310.519157:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762310.519160:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1744 00000100:00080000:0.0:1789762310.519164:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762310.519168:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789762310.519172:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762310.519181:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d1f80 x1876700024664448/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789762310.519229:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789762310.519276:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762310/real 1789762310] req@ffffa06c091d0a80 x1876700024664192/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762335 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762310.519291:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0a80 x1876700024664192/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762335 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762310.519299:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762310.519302:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789762310.519311:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024664192:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789762310.519334:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024664320:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789762310.519352:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789762310.519361:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762310/real 1789762310] req@ffffa06c091d0700 x1876700024664320/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762335 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762310.519369:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0700 x1876700024664320/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762335 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762310.519376:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762310.519379:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789762310.519383:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024664320:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789762310.519403:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024664448:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789762310.519425:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789762310.519450:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762310/real 1789762310] req@ffffa06c091d1f80 x1876700024664448/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762335 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762310.519459:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d1f80 x1876700024664448/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762335 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762310.519465:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762310.519467:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789762310.519469:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024664448:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789762310.774998:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762311.030974:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789762311.798973:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762312.055001:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762312.823049:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762313.079016:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762313.846975:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762314.102993:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789762314.871040:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762315.126974:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789762315.639002:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762315.639016:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789762315.639021:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762315.639033:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1749 00000100:00080000:2.0:1789762315.639064:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762315.639074:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789762315.639143:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51f80 x1876700024664576/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762315.639172:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762315.639176:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762315.639195:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762315.639200:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1749 00000100:00080000:2.0:1789762315.639205:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762315.639209:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789762315.639224:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762315.639259:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024664576:192.168.203.107@tcp:250:kworker.0 00000100:00100000:2.0:1789762315.639269:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50380 x1876700024664704/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762315.639287:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762315.639291:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762315.639294:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762315.639299:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1749 00000100:00080000:2.0:1789762315.639303:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762315.639308:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789762315.639313:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762315.639325:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e53b80 x1876700024664832/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789762315.639343:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762315.639415:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762315/real 1789762315] req@ffffa06c07e51f80 x1876700024664576/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762340 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762315.639441:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51f80 x1876700024664576/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762340 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762315.639453:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762315.639460:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762315.639472:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024664576:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789762315.639510:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024664704:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789762315.639543:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762315.639560:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762315/real 1789762315] req@ffffa06c07e50380 x1876700024664704/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762340 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762315.639584:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50380 x1876700024664704/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762340 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762315.639596:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762315.639600:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762315.639606:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024664704:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789762315.639615:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e53b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024664832:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789762315.639641:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762315.639659:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762315/real 1789762315] req@ffffa06c07e53b80 x1876700024664832/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762340 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762315.639673:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e53b80 x1876700024664832/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762340 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762315.639696:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762315.639701:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762315.639705:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e53b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024664832:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789762315.895006:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762316.150991:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762316.919027:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762317.174984:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789762317.943067:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762318.198910:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762318.967044:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762319.222917:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762319.991089:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762320.246991:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789762320.758984:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762320.758998:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789762320.759004:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762320.759016:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1754 00000100:00080000:0.0:1789762320.759027:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762320.759035:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789762320.759081:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d1f80 x1876700024664960/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762320.759108:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762320.759111:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762320.759113:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762320.759116:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1754 00000100:00080000:0.0:1789762320.759119:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762320.759122:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789762320.759132:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762320.759146:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0700 x1876700024665088/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789762320.759146:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024664960:192.168.203.107@tcp:250:kworker.0 00000100:00080000:0.0:1789762320.759162:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762320.759164:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762320.759165:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762320.759167:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1754 00000100:00080000:0.0:1789762320.759169:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762320.759172:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789762320.759196:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762320.759205:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0a80 x1876700024665216/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789762320.759228:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762320.759275:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762320/real 1789762320] req@ffffa06c091d1f80 x1876700024664960/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762345 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762320.759292:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d1f80 x1876700024664960/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762345 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762320.759301:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762320.759304:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762320.759307:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024664960:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789762320.759358:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024665088:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789762320.759393:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762320.759404:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762320/real 1789762320] req@ffffa06c091d0700 x1876700024665088/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762345 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762320.759414:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0700 x1876700024665088/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762345 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762320.759428:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762320.759431:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762320.759434:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024665088:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789762320.759438:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024665216:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789762320.759454:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762320.759462:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762320/real 1789762320] req@ffffa06c091d0a80 x1876700024665216/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762345 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762320.759471:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0a80 x1876700024665216/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762345 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762320.759477:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762320.759479:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762320.759481:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024665216:192.168.203.107@tcp:38:kworker.0 00010000:00010000:1.0:1789762321.015027:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762321.270946:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762322.039048:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762322.294981:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789762323.063017:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762323.318937:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789762324.087071:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762324.342997:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762325.111017:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762325.366920:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:3.0:1789762325.878973:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762325.878984:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789762325.878988:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762325.878999:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1759 00000100:00080000:3.0:1789762325.879012:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762325.879020:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:3.0:1789762325.879062:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872f800 x1876700024665344/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762325.879088:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762325.879091:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762325.879092:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762325.879095:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1759 00000100:00080000:3.0:1789762325.879098:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762325.879100:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789762325.879109:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762325.879122:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872df80 x1876700024665472/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789762325.879130:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024665344:192.168.203.107@tcp:250:kworker.0 00000100:00080000:3.0:1789762325.879135:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762325.879138:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762325.879139:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762325.879161:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1759 00000100:00080000:3.0:1789762325.879165:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762325.879168:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789762325.879173:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762325.879182:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872d880 x1876700024665600/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789762325.879209:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762325.879252:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762325/real 1789762325] req@ffffa06c1872f800 x1876700024665344/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762350 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762325.879291:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872f800 x1876700024665344/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762350 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762325.879311:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762325.879315:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762325.879318:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024665344:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789762325.879343:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024665472:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789762325.879362:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762325.879381:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762325/real 1789762325] req@ffffa06c1872df80 x1876700024665472/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762350 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762325.879390:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872df80 x1876700024665472/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762350 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762325.879397:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762325.879399:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762325.879401:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024665472:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789762325.879405:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024665600:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789762325.879416:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762325.879422:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762325/real 1789762325] req@ffffa06c1872d880 x1876700024665600/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762350 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762325.879430:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872d880 x1876700024665600/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762350 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762325.879445:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762325.879448:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762325.879451:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024665600:192.168.203.107@tcp:38:kworker.0 00010000:00010000:1.0:1789762326.135075:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762326.390976:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789762327.158990:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762327.415071:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762328.183067:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762328.438970:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789762329.206950:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762329.462908:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789762330.231076:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762330.486950:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789762330.998942:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762330.998950:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789762330.998953:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762330.998962:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1764 00000100:00080000:0.0:1789762330.998970:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762330.998976:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789762330.999007:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0380 x1876700024665728/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762330.999028:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762330.999031:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762330.999032:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762330.999035:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1764 00000100:00080000:0.0:1789762330.999037:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762330.999040:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789762330.999046:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762330.999054:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d2680 x1876700024665856/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762330.999063:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762330.999065:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762330.999066:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762330.999069:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1764 00000100:00100000:1.0:1789762330.999080:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024665728:192.168.203.107@tcp:250:kworker.0 00000100:00080000:0.0:1789762330.999083:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762330.999090:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789762330.999094:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762330.999104:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d3100 x1876700024665984/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789762330.999138:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762330.999171:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762330/real 1789762330] req@ffffa06c091d0380 x1876700024665728/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762355 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762330.999183:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0380 x1876700024665728/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762355 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762330.999191:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762330.999194:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762330.999197:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024665728:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789762330.999202:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024665856:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789762330.999215:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762330.999230:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762330/real 1789762330] req@ffffa06c091d2680 x1876700024665856/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762355 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762330.999255:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d2680 x1876700024665856/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762355 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762330.999261:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762330.999263:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762330.999265:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024665856:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789762330.999284:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024665984:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789762330.999298:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762330.999305:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762330/real 1789762330] req@ffffa06c091d3100 x1876700024665984/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762355 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762330.999313:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d3100 x1876700024665984/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762355 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762330.999317:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762330.999319:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762330.999321:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024665984:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789762331.254976:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762331.510927:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762332.279076:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762332.534962:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762333.303055:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762333.558972:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762334.327007:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762334.582912:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762335.350968:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762335.606926:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:1.0:1789762336.118943:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762336.118952:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789762336.118973:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762336.118982:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1769 00000100:00080000:1.0:1789762336.118993:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762336.118999:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789762336.119032:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2c000 x1876700024666112/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762336.119061:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762336.119065:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762336.119067:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762336.119071:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1769 00000100:00080000:1.0:1789762336.119074:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762336.119078:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789762336.119087:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762336.119101:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2c700 x1876700024666240/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762336.119113:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762336.119118:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762336.119121:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1789762336.119121:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024666112:192.168.203.107@tcp:250:kworker.0 00000100:00080000:1.0:1789762336.119131:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1769 00000100:00080000:1.0:1789762336.119136:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762336.119140:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789762336.119146:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762336.119159:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2ca80 x1876700024666368/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789762336.119206:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789762336.119250:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762336/real 1789762336] req@ffffa06c08e2c000 x1876700024666112/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762361 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762336.119270:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2c000 x1876700024666112/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762361 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762336.119281:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762336.119286:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789762336.119290:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024666112:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789762336.119314:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024666240:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789762336.119338:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789762336.119352:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762336/real 1789762336] req@ffffa06c08e2c700 x1876700024666240/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762361 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762336.119382:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2c700 x1876700024666240/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762361 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762336.119392:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762336.119396:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789762336.119400:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024666240:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789762336.119407:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024666368:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789762336.119432:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789762336.119444:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762336/real 1789762336] req@ffffa06c08e2ca80 x1876700024666368/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762361 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762336.119458:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2ca80 x1876700024666368/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762361 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762336.119467:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762336.119471:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789762336.119475:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024666368:192.168.203.107@tcp:38:kworker.0 00010000:00010000:2.0:1789762336.375101:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762336.630964:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762337.399060:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762337.655051:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762338.423029:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762338.678943:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762339.446981:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762339.703027:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762340.470993:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762340.726912:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789762341.238931:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762341.238938:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789762341.238942:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762341.238950:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1775 00000100:00080000:2.0:1789762341.238959:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762341.238965:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789762341.238996:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50e00 x1876700024666496/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762341.239017:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762341.239020:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762341.239021:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762341.239024:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1775 00000100:00080000:2.0:1789762341.239026:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762341.239028:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789762341.239035:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762341.239044:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51500 x1876700024666624/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762341.239053:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762341.239056:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762341.239073:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1789762341.239081:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024666496:192.168.203.107@tcp:250:kworker.0 00000100:00080000:2.0:1789762341.239082:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1775 00000100:00080000:2.0:1789762341.239090:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762341.239113:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789762341.239118:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762341.239149:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50000 x1876700024666752/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789762341.239221:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789762341.239266:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762341/real 1789762341] req@ffffa06c07e50e00 x1876700024666496/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762366 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762341.239300:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50e00 x1876700024666496/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762366 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762341.239308:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762341.239311:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789762341.239314:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024666496:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789762341.239333:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024666624:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789762341.239351:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789762341.239360:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762341/real 1789762341] req@ffffa06c07e51500 x1876700024666624/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762366 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762341.239368:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51500 x1876700024666624/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762366 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762341.239373:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762341.239375:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789762341.239377:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024666624:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789762341.239381:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024666752:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789762341.239391:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789762341.239397:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762341/real 1789762341] req@ffffa06c07e50000 x1876700024666752/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762366 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762341.239404:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50000 x1876700024666752/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762366 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762341.239419:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762341.239422:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789762341.239424:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024666752:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789762341.495010:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762341.751042:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762342.518955:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762342.774923:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762343.542980:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762343.798984:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762344.567008:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762344.823050:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789762345.591001:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762345.846965:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789762346.358961:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762346.358974:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789762346.358980:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762346.358992:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1780 00000100:00080000:2.0:1789762346.359006:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762346.359015:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789762346.359070:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e52300 x1876700024666880/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762346.359138:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762346.359146:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762346.359150:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762346.359159:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1780 00000100:00080000:2.0:1789762346.359167:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762346.359175:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789762346.359188:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762346.359210:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51880 x1876700024667008/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789762346.359217:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e52300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024666880:192.168.203.107@tcp:250:kworker.0 00000100:00080000:2.0:1789762346.359243:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762346.359249:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762346.359271:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762346.359279:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1780 00000100:00080000:2.0:1789762346.359308:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762346.359314:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789762346.359323:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1789762346.359327:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789762346.359349:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51180 x1876700024667136/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789762346.359403:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762346/real 1789762346] req@ffffa06c07e52300 x1876700024666880/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762371 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762346.359429:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e52300 x1876700024666880/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762371 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762346.359445:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762346.359453:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789762346.359461:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e52300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024666880:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789762346.359518:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024667008:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789762346.359563:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789762346.359597:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762346/real 1789762346] req@ffffa06c07e51880 x1876700024667008/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762371 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762346.359624:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51880 x1876700024667008/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762371 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762346.359668:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762346.359676:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789762346.359684:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024667008:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789762346.359714:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024667136:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789762346.359760:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789762346.359786:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762346/real 1789762346] req@ffffa06c07e51180 x1876700024667136/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762371 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762346.359811:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51180 x1876700024667136/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762371 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762346.359823:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762346.359827:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789762346.359832:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024667136:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789762346.615114:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762346.871017:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789762347.639003:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762347.894940:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789762348.663011:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762348.918961:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762349.686964:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762349.942935:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789762350.711104:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762350.967002:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789762351.478925:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762351.478933:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789762351.478937:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762351.478946:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1785 00000100:00080000:2.0:1789762351.478957:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762351.478971:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789762351.479008:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51180 x1876700024667264/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762351.479033:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762351.479036:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762351.479038:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762351.479041:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1785 00000100:00080000:2.0:1789762351.479044:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762351.479047:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789762351.479057:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762351.479070:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51880 x1876700024667392/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789762351.479098:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024667264:192.168.203.107@tcp:250:kworker.0 00000100:00080000:2.0:1789762351.479130:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762351.479134:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762351.479137:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762351.479143:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1785 00000100:00080000:2.0:1789762351.479149:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762351.479155:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789762351.479164:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762351.479176:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e52300 x1876700024667520/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789762351.479236:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024667392:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789762351.479277:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e52300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024667520:192.168.203.107@tcp:38:kworker.0 00010000:00010000:1.0:1789762351.735021:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762351.990916:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762352.759095:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762353.014957:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789762353.783043:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762354.038950:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000400:00000100:0.0:1789762354.551266:0:2329:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.107:988 00000400:00000100:0.0:1789762354.551310:0:2329:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.107@tcp at host 192.168.203.107 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1789762354.551338:0:2329:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.107@tcp 00000800:00000100:0.0:1789762354.551384:0:2329:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.107@tcp 00000800:00000100:0.0:1789762354.551400:0:2329:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.107@tcp 00000100:00100000:1.0:1789762354.551462:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762351/real 1789762354] req@ffffa06c07e51880 x1876700024667392/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762376 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762354.551517:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51880 x1876700024667392/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762376 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762354.551548:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762354.551555:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762354.551560:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024667392:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789762354.551586:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762351/real 1789762354] req@ffffa06c07e52300 x1876700024667520/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762376 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762354.551601:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e52300 x1876700024667520/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762376 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762354.551610:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762354.551614:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762354.551618:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e52300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024667520:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789762354.551627:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762351/real 1789762354] req@ffffa06c07e51180 x1876700024667264/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762376 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762354.551641:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51180 x1876700024667264/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762376 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762354.551650:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762354.551654:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762354.551658:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024667264:192.168.203.107@tcp:250:kworker.0 00010000:00010000:1.0:1789762354.807085:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762355.062962:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762355.831022:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762356.086974:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:3.0:1789762356.598962:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762356.598978:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789762356.598984:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762356.598994:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1790 00000100:00080000:3.0:1789762356.599003:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762356.599009:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:3.0:1789762356.599062:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872c380 x1876700024667648/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762356.599086:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762356.599089:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762356.599090:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762356.599093:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1790 00000100:00080000:3.0:1789762356.599097:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762356.599099:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789762356.599112:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762356.599128:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872e300 x1876700024667776/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789762356.599135:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024667648:192.168.203.107@tcp:250:kworker.0 00000100:00080000:3.0:1789762356.599149:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762356.599156:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762356.599160:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762356.599167:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1790 00000100:00080000:3.0:1789762356.599170:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762356.599173:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789762356.599177:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762356.599187:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872d180 x1876700024667904/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789762356.599243:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762356.599283:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762356/real 1789762356] req@ffffa06c1872c380 x1876700024667648/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762381 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762356.599301:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872c380 x1876700024667648/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762381 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762356.599312:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762356.599317:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762356.599321:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024667648:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789762356.599345:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024667776:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789762356.599370:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762356.599383:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762356/real 1789762356] req@ffffa06c1872e300 x1876700024667776/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762381 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762356.599398:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872e300 x1876700024667776/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762381 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762356.599408:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762356.599411:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762356.599415:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024667776:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789762356.599428:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024667904:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789762356.599451:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762356.599463:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762356/real 1789762356] req@ffffa06c1872d180 x1876700024667904/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762381 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762356.599489:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872d180 x1876700024667904/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762381 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762356.599496:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762356.599499:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762356.599503:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024667904:192.168.203.107@tcp:38:kworker.0 00010000:00010000:1.0:1789762356.855014:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762357.110956:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789762357.879005:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762358.134971:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762358.903017:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762359.158973:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789762359.927100:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762360.182937:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789762360.951062:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762361.206932:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:3.0:1789762361.719024:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762361.719036:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789762361.719083:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762361.719095:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1795 00000100:00080000:3.0:1789762361.719109:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762361.719116:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:3.0:1789762361.719160:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872ce00 x1876700024668032/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762361.719193:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762361.719197:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762361.719198:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762361.719202:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1795 00000100:00080000:3.0:1789762361.719205:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762361.719208:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789762361.719223:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762361.719235:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872ca80 x1876700024668160/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762361.719258:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762361.719262:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762361.719263:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762361.719269:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1795 00000100:00100000:1.0:1789762361.719271:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024668032:192.168.203.107@tcp:250:kworker.0 00000100:00080000:3.0:1789762361.719273:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762361.719281:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789762361.719286:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762361.719296:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872f100 x1876700024668288/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789762361.719378:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762361.719438:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762361/real 1789762361] req@ffffa06c1872ce00 x1876700024668032/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762386 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762361.719455:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872ce00 x1876700024668032/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762386 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762361.719477:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762361.719480:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762361.719485:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024668032:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789762361.719522:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024668160:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789762361.719552:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762361.719562:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762361/real 1789762361] req@ffffa06c1872ca80 x1876700024668160/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762386 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762361.719573:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872ca80 x1876700024668160/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762386 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762361.719580:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762361.719583:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762361.719586:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024668160:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789762361.719590:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024668288:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789762361.719601:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762361.719619:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762361/real 1789762361] req@ffffa06c1872f100 x1876700024668288/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762386 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762361.719628:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872f100 x1876700024668288/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762386 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762361.719634:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762361.719636:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762361.719638:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024668288:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789762361.975053:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762362.230927:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789762362.999111:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762363.254940:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789762364.023018:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762364.278974:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762365.047096:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762365.302988:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789762366.071077:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762366.326994:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789762366.839006:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762366.839016:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789762366.839020:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762366.839030:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1800 00000100:00080000:2.0:1789762366.839040:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762366.839048:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789762366.839086:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50000 x1876700024668416/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762366.839111:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762366.839114:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762366.839115:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762366.839118:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1800 00000100:00080000:2.0:1789762366.839121:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762366.839124:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789762366.839145:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762366.839156:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51500 x1876700024668544/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762366.839165:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1789762366.839168:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024668416:192.168.203.107@tcp:250:kworker.0 00000100:00080000:2.0:1789762366.839169:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762366.839173:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762366.839178:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1800 00000100:00080000:2.0:1789762366.839181:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762366.839184:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789762366.839187:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762366.839199:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50e00 x1876700024668672/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789762366.839228:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762366.839261:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762366/real 1789762366] req@ffffa06c07e50000 x1876700024668416/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762391 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762366.839274:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50000 x1876700024668416/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762391 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762366.839282:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762366.839285:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762366.839288:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024668416:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789762366.839306:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024668544:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789762366.839319:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762366.839326:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762366/real 1789762366] req@ffffa06c07e51500 x1876700024668544/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762391 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762366.839334:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51500 x1876700024668544/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762391 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762366.839339:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762366.839342:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762366.839344:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024668544:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789762366.839347:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024668672:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789762366.839357:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762366.839362:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762366/real 1789762366] req@ffffa06c07e50e00 x1876700024668672/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762391 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762366.839377:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50e00 x1876700024668672/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762391 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762366.839383:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762366.839384:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762366.839386:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024668672:192.168.203.107@tcp:38:kworker.0 00010000:00010000:0.0:1789762367.095075:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762367.350942:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762368.119206:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762368.375022:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762369.142945:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762369.398931:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762370.166997:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762370.422918:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762371.191029:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762371.446987:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789762371.958914:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762371.958922:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789762371.958937:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762371.958947:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1805 00000100:00080000:2.0:1789762371.958966:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762371.958972:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789762371.959001:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e53b80 x1876700024668800/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762371.959021:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762371.959024:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762371.959035:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762371.959039:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1805 00000100:00080000:2.0:1789762371.959043:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762371.959046:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789762371.959054:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762371.959059:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e53b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024668800:192.168.203.107@tcp:250:kworker.0 00000100:00100000:2.0:1789762371.959068:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50380 x1876700024668928/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762371.959079:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762371.959081:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762371.959082:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762371.959085:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1805 00000100:00080000:2.0:1789762371.959088:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762371.959090:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789762371.959094:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762371.959101:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51f80 x1876700024669056/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789762371.959126:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762371.959157:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762371/real 1789762371] req@ffffa06c07e53b80 x1876700024668800/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762396 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762371.959170:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e53b80 x1876700024668800/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762396 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762371.959178:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762371.959181:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762371.959184:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e53b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024668800:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789762371.959200:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024668928:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789762371.959214:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762371.959222:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762371/real 1789762371] req@ffffa06c07e50380 x1876700024668928/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762396 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762371.959237:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50380 x1876700024668928/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762396 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762371.959243:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762371.959246:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762371.959248:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024668928:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789762371.959252:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024669056:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789762371.959264:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762371.959270:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762371/real 1789762371] req@ffffa06c07e51f80 x1876700024669056/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762396 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762371.959278:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51f80 x1876700024669056/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762396 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762371.959283:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762371.959284:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762371.959286:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024669056:192.168.203.107@tcp:38:kworker.0 00010000:00010000:1.0:1789762372.214991:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762372.470941:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762373.239086:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762373.494964:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762374.262967:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762374.519010:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762375.287009:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762375.542930:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789762376.311017:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762376.566954:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:1.0:1789762377.078953:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762377.078976:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789762377.078979:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762377.078987:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1810 00000100:00080000:1.0:1789762377.078996:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762377.079002:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789762377.079030:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2d180 x1876700024669184/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762377.079053:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762377.079056:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762377.079057:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762377.079060:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1810 00000100:00080000:1.0:1789762377.079062:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762377.079065:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789762377.079072:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762377.079082:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2ca80 x1876700024669312/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762377.079091:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762377.079094:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762377.079097:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1789762377.079099:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024669184:192.168.203.107@tcp:250:kworker.0 00000100:00080000:1.0:1789762377.079102:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1810 00000100:00080000:1.0:1789762377.079107:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762377.079110:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789762377.079113:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762377.079121:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2c700 x1876700024669440/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789762377.079158:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789762377.079192:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762377/real 1789762377] req@ffffa06c08e2d180 x1876700024669184/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762402 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762377.079206:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2d180 x1876700024669184/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762402 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762377.079214:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762377.079217:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789762377.079220:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024669184:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789762377.079249:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024669312:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789762377.079263:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789762377.079274:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762377/real 1789762377] req@ffffa06c08e2ca80 x1876700024669312/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762402 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762377.079287:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2ca80 x1876700024669312/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762402 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762377.079296:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762377.079299:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789762377.079312:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024669312:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789762377.079319:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024669440:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789762377.079340:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789762377.079348:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762377/real 1789762377] req@ffffa06c08e2c700 x1876700024669440/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762402 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762377.079356:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2c700 x1876700024669440/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762402 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762377.079362:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762377.079364:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789762377.079366:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024669440:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789762377.335026:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762377.590960:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762378.359066:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762378.614975:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789762379.382976:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762379.638969:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762380.407071:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762380.663017:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762381.430983:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762381.686972:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789762382.199029:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762382.199038:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789762382.199042:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762382.199050:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1816 00000100:00080000:0.0:1789762382.199060:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762382.199083:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789762382.199130:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d1500 x1876700024669568/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762382.199162:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762382.199185:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762382.199188:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762382.199192:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1816 00000100:00080000:0.0:1789762382.199197:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762382.199200:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789762382.199209:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762382.199213:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024669568:192.168.203.107@tcp:250:kworker.0 00000100:00100000:0.0:1789762382.199233:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d2a00 x1876700024669696/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762382.199245:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762382.199247:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762382.199249:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762382.199251:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1816 00000100:00080000:0.0:1789762382.199254:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762382.199256:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789762382.199260:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762382.199269:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0e00 x1876700024669824/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789762382.199293:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789762382.199343:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762382/real 1789762382] req@ffffa06c091d1500 x1876700024669568/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762407 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762382.199357:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d1500 x1876700024669568/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762407 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762382.199365:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762382.199368:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789762382.199371:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024669568:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789762382.199399:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024669696:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789762382.199417:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789762382.199425:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762382/real 1789762382] req@ffffa06c091d2a00 x1876700024669696/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762407 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762382.199434:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d2a00 x1876700024669696/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762407 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762382.199440:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762382.199442:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789762382.199445:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024669696:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789762382.199448:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024669824:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789762382.199459:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789762382.199465:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762382/real 1789762382] req@ffffa06c091d0e00 x1876700024669824/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762407 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762382.199473:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0e00 x1876700024669824/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762407 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762382.199478:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762382.199480:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789762382.199482:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024669824:192.168.203.107@tcp:38:kworker.0 00010000:00010000:1.0:1789762382.454982:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762382.710934:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762383.478990:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762383.734934:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762384.502976:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762384.758947:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789762385.526963:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762385.782930:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762386.550959:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762386.806949:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789762387.318948:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762387.318956:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789762387.318959:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762387.318968:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1821 00000100:00080000:0.0:1789762387.318976:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762387.318983:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789762387.319014:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0000 x1876700024669952/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762387.319050:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762387.319053:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762387.319054:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762387.319058:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1821 00000100:00080000:0.0:1789762387.319061:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762387.319064:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789762387.319071:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762387.319083:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d3b80 x1876700024670080/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762387.319092:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762387.319095:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762387.319097:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762387.319101:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1821 00000100:00080000:0.0:1789762387.319104:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1789762387.319108:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024669952:192.168.203.107@tcp:250:kworker.0 00000100:00080000:0.0:1789762387.319110:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789762387.319119:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762387.319129:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d2300 x1876700024670208/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789762387.319169:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789762387.319396:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762387/real 1789762387] req@ffffa06c091d0000 x1876700024669952/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762412 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762387.319413:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0000 x1876700024669952/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762412 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762387.319422:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762387.319462:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789762387.319467:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024669952:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789762387.319476:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024670080:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789762387.319496:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789762387.319507:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762387/real 1789762387] req@ffffa06c091d3b80 x1876700024670080/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762412 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762387.319515:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d3b80 x1876700024670080/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762412 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762387.319521:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762387.319523:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789762387.319525:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024670080:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789762387.319542:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024670208:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789762387.319554:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789762387.319561:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762387/real 1789762387] req@ffffa06c091d2300 x1876700024670208/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762412 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762387.319568:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d2300 x1876700024670208/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762412 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762387.319583:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762387.319585:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789762387.319587:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024670208:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789762387.574965:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762387.830930:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789762388.599031:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762388.854940:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762389.623031:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762389.878936:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762390.647009:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762390.902977:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762391.671069:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762391.926973:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:1.0:1789762392.439019:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762392.439032:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789762392.439037:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762392.439050:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1826 00000100:00080000:1.0:1789762392.439081:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762392.439090:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789762392.439145:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2c000 x1876700024670336/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762392.439179:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762392.439220:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762392.439223:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762392.439228:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1826 00000100:00080000:1.0:1789762392.439233:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762392.439239:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789762392.439251:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762392.439273:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024670336:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789762392.439281:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2e300 x1876700024670464/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762392.439296:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762392.439300:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762392.439302:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762392.439306:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1826 00000100:00080000:1.0:1789762392.439309:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762392.439312:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789762392.439318:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762392.439330:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2ed80 x1876700024670592/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789762392.439367:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789762392.439431:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762392/real 1789762392] req@ffffa06c08e2c000 x1876700024670336/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762417 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762392.439451:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2c000 x1876700024670336/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762417 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762392.439461:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762392.439466:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789762392.439470:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024670336:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789762392.439504:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024670464:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789762392.439527:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789762392.439540:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762392/real 1789762392] req@ffffa06c08e2e300 x1876700024670464/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762417 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762392.439551:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2e300 x1876700024670464/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762417 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762392.439574:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762392.439578:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789762392.439581:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024670464:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789762392.439587:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024670592:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789762392.439608:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789762392.439618:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762392/real 1789762392] req@ffffa06c08e2ed80 x1876700024670592/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762417 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762392.439628:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2ed80 x1876700024670592/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762417 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762392.439635:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762392.439638:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789762392.439640:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024670592:192.168.203.107@tcp:38:kworker.0 00010000:00010000:2.0:1789762392.695007:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762392.950928:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762393.719168:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762393.974993:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762394.742948:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762394.998977:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762395.766979:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762396.022942:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762396.791098:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762397.046938:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789762397.558965:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762397.558975:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789762397.558994:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762397.559004:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1831 00000100:00080000:2.0:1789762397.559016:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762397.559024:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789762397.559057:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e52680 x1876700024670720/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762397.559084:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762397.559087:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762397.559090:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762397.559094:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1831 00000100:00080000:2.0:1789762397.559098:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762397.559103:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789762397.559113:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762397.559128:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e53800 x1876700024670848/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762397.559142:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762397.559147:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762397.559149:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762397.559153:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1831 00000100:00080000:2.0:1789762397.559159:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762397.559166:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789762397.559172:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762397.559179:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e52680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024670720:192.168.203.107@tcp:250:kworker.0 00000100:00100000:2.0:1789762397.559192:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51c00 x1876700024670976/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789762397.559268:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789762397.559323:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762397/real 1789762397] req@ffffa06c07e52680 x1876700024670720/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762422 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762397.559341:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e52680 x1876700024670720/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762422 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762397.559367:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762397.559370:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789762397.559374:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e52680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024670720:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789762397.559380:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e53800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024670848:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789762397.559409:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789762397.559419:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762397/real 1789762397] req@ffffa06c07e53800 x1876700024670848/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762422 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762397.559428:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e53800 x1876700024670848/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762422 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762397.559434:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762397.559436:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789762397.559439:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e53800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024670848:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789762397.559462:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024670976:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789762397.559475:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789762397.559481:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762397/real 1789762397] req@ffffa06c07e51c00 x1876700024670976/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762422 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762397.559489:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51c00 x1876700024670976/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762422 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762397.559494:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762397.559496:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789762397.559498:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024670976:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789762397.814969:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762398.070928:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762398.838948:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762399.094912:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762399.863053:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762400.118996:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762400.887045:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762401.142952:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762401.911005:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762402.166915:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789762402.678952:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762402.678966:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789762402.678971:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762402.678984:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1836 00000100:00080000:0.0:1789762402.679030:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762402.679041:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789762402.679083:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d2300 x1876700024671104/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762402.679110:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762402.679129:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762402.679131:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762402.679135:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1836 00000100:00080000:0.0:1789762402.679140:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762402.679144:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789762402.679157:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762402.679169:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d3b80 x1876700024671232/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762402.679182:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762402.679186:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:3.0:1789762402.679186:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024671104:192.168.203.107@tcp:250:kworker.0 00000100:00080000:0.0:1789762402.679190:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762402.679199:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1836 00000100:00080000:0.0:1789762402.679205:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762402.679210:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789762402.679214:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762402.679225:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0000 x1876700024671360/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789762402.679278:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789762402.679334:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762402/real 1789762402] req@ffffa06c091d2300 x1876700024671104/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762427 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762402.679353:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d2300 x1876700024671104/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762427 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762402.679362:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762402.679365:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789762402.679368:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024671104:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789762402.679388:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024671232:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789762402.679415:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789762402.679423:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762402/real 1789762402] req@ffffa06c091d3b80 x1876700024671232/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762427 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762402.679431:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d3b80 x1876700024671232/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762427 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762402.679437:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762402.679439:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789762402.679441:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024671232:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789762402.679460:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024671360:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789762402.679487:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789762402.679494:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762402/real 1789762402] req@ffffa06c091d0000 x1876700024671360/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762427 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762402.679501:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0000 x1876700024671360/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762427 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762402.679507:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762402.679509:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789762402.679511:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024671360:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789762402.934979:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762403.190950:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762403.958994:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762404.214922:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762404.983042:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762405.238968:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762406.006940:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762406.262944:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762407.031014:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762407.286925:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789762407.798948:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762407.798956:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789762407.798960:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762407.798969:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1841 00000100:00080000:2.0:1789762407.798978:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762407.798984:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789762407.799015:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e53480 x1876700024671488/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762407.799039:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762407.799041:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762407.799043:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762407.799046:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1841 00000100:00080000:2.0:1789762407.799048:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762407.799050:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789762407.799057:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762407.799067:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50a80 x1876700024671616/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762407.799076:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762407.799079:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762407.799098:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1789762407.799099:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e53480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024671488:192.168.203.107@tcp:250:kworker.0 00000100:00080000:2.0:1789762407.799113:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1841 00000100:00080000:2.0:1789762407.799118:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762407.799122:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789762407.799127:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762407.799141:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50700 x1876700024671744/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789762407.799194:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789762407.799242:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762407/real 1789762407] req@ffffa06c07e53480 x1876700024671488/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762432 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762407.799258:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e53480 x1876700024671488/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762432 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762407.799268:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762407.799272:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789762407.799275:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e53480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024671488:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789762407.799282:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024671616:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789762407.799348:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789762407.799363:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762407/real 1789762407] req@ffffa06c07e50a80 x1876700024671616/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762432 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762407.799376:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50a80 x1876700024671616/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762432 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762407.799385:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762407.799389:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789762407.799393:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024671616:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789762407.799423:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024671744:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789762407.799441:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789762407.799460:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762407/real 1789762407] req@ffffa06c07e50700 x1876700024671744/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762432 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762407.799473:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50700 x1876700024671744/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762432 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762407.799482:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762407.799485:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789762407.799489:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024671744:192.168.203.107@tcp:38:kworker.0 00010000:00010000:2.0:1789762408.055013:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762408.310940:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762409.079005:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762409.334904:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762410.102952:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762410.358944:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789762411.127041:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762411.382967:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762412.150952:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762412.406913:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789762412.918950:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762412.918973:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789762412.918976:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762412.918984:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1846 00000100:00080000:0.0:1789762412.918994:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762412.919006:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789762412.919034:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0e00 x1876700024671872/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762412.919054:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762412.919057:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762412.919059:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762412.919062:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1846 00000100:00080000:0.0:1789762412.919064:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762412.919066:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789762412.919075:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762412.919083:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d2a00 x1876700024672000/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762412.919092:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762412.919095:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762412.919096:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1789762412.919101:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024671872:192.168.203.107@tcp:250:kworker.0 00000100:00080000:0.0:1789762412.919102:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1846 00000100:00080000:0.0:1789762412.919108:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762412.919112:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789762412.919115:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762412.919127:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d1500 x1876700024672128/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789762412.919162:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789762412.919204:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762412/real 1789762412] req@ffffa06c091d0e00 x1876700024671872/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762437 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762412.919218:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0e00 x1876700024671872/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762437 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762412.919226:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762412.919229:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789762412.919232:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024671872:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789762412.919250:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024672000:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789762412.919264:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789762412.919272:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762412/real 1789762412] req@ffffa06c091d2a00 x1876700024672000/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762437 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762412.919280:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d2a00 x1876700024672000/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762437 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762412.919285:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762412.919287:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789762412.919289:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024672000:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789762412.919292:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024672128:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789762412.919302:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789762412.919308:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762412/real 1789762412] req@ffffa06c091d1500 x1876700024672128/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762437 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762412.919315:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d1500 x1876700024672128/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762437 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762412.919329:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762412.919332:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789762412.919334:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024672128:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789762413.174964:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762413.430941:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789762414.198985:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762414.454918:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762415.222958:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762415.478949:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762416.246970:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762416.502921:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789762417.270976:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762417.526955:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789762418.038915:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762418.038924:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789762418.038928:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762418.038936:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1851 00000100:00080000:0.0:1789762418.038945:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762418.038951:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789762418.038983:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d3100 x1876700024672256/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762418.039009:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762418.039014:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762418.039016:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762418.039022:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1851 00000100:00080000:0.0:1789762418.039028:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762418.039031:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:3.0:1789762418.039038:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024672256:192.168.203.107@tcp:250:kworker.0 00000008:00100000:0.0:1789762418.039040:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762418.039061:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d2680 x1876700024672384/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762418.039072:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762418.039075:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762418.039076:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762418.039079:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1851 00000100:00080000:0.0:1789762418.039093:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762418.039096:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789762418.039100:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762418.039110:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0380 x1876700024672512/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789762418.039138:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024672384:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789762418.039179:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024672512:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789762418.294962:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762418.550942:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762419.318970:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762419.574947:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789762420.343061:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762420.599002:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000400:00000100:2.0:1789762421.111079:0:2330:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.107:988 00000400:00000100:2.0:1789762421.111125:0:2330:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.107@tcp at host 192.168.203.107 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1789762421.111143:0:2330:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.107@tcp 00000800:00000100:2.0:1789762421.111179:0:2330:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.107@tcp 00000800:00000100:2.0:1789762421.111186:0:2330:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.107@tcp 00000100:00100000:3.0:1789762421.111236:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762418/real 1789762421] req@ffffa06c091d2680 x1876700024672384/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762443 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762421.111276:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d2680 x1876700024672384/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762443 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762421.111292:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762421.111299:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789762421.111305:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024672384:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789762421.111315:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762418/real 1789762421] req@ffffa06c091d0380 x1876700024672512/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762443 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762421.111330:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0380 x1876700024672512/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762443 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762421.111357:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762421.111363:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789762421.111367:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024672512:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789762421.111377:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762418/real 1789762421] req@ffffa06c091d3100 x1876700024672256/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762443 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762421.111393:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d3100 x1876700024672256/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762443 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762421.111402:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762421.111406:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789762421.111409:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024672256:192.168.203.107@tcp:250:kworker.0 00010000:00010000:0.0:1789762421.367041:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762421.622951:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762422.391091:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762422.646974:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789762423.159048:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762423.159066:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789762423.159073:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762423.159104:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1856 00000100:00080000:0.0:1789762423.159125:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762423.159138:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789762423.159214:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0a80 x1876700024672640/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762423.159269:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762423.159274:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762423.159276:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762423.159282:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1856 00000100:00080000:0.0:1789762423.159287:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762423.159292:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789762423.159306:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762423.159334:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0700 x1876700024672768/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762423.159386:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762423.159392:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762423.159395:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1789762423.159402:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024672640:192.168.203.107@tcp:250:kworker.0 00000100:00080000:0.0:1789762423.159405:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1856 00000100:00080000:0.0:1789762423.159417:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762423.159427:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789762423.159437:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762423.159457:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d1f80 x1876700024672896/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789762423.159513:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789762423.159608:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762423/real 1789762423] req@ffffa06c091d0a80 x1876700024672640/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762448 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762423.159639:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0a80 x1876700024672640/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762448 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762423.159664:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762423.159672:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789762423.159679:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024672640:192.168.203.107@tcp:250:kworker.0 00000100:00100000:2.0:1789762423.159738:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024672768:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789762423.159799:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789762423.159824:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762423/real 1789762423] req@ffffa06c091d0700 x1876700024672768/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762448 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762423.159895:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0700 x1876700024672768/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762448 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762423.159913:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762423.159920:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789762423.159927:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024672768:192.168.203.107@tcp:38:kworker.0 00000100:00100000:2.0:1789762423.159941:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024672896:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789762423.159982:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789762423.160004:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762423/real 1789762423] req@ffffa06c091d1f80 x1876700024672896/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762448 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762423.160025:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d1f80 x1876700024672896/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762448 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762423.160038:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762423.160044:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789762423.160049:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024672896:192.168.203.107@tcp:38:kworker.0 00010000:00010000:0.0:1789762423.415154:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762423.671003:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762424.439073:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762424.694999:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762425.462982:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762425.718976:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789762426.487047:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762426.742954:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762427.511020:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762427.766981:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:3.0:1789762428.278975:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762428.278984:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789762428.278989:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762428.278998:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1862 00000100:00080000:3.0:1789762428.279008:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762428.279015:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:3.0:1789762428.279048:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872c700 x1876700024673024/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762428.279079:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762428.279082:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762428.279097:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762428.279101:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1862 00000100:00080000:3.0:1789762428.279105:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762428.279109:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789762428.279118:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762428.279131:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872dc00 x1876700024673152/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789762428.279136:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024673024:192.168.203.107@tcp:250:kworker.0 00000100:00080000:3.0:1789762428.279150:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762428.279154:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762428.279155:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762428.279159:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1862 00000100:00080000:3.0:1789762428.279162:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762428.279164:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789762428.279168:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762428.279179:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872ed80 x1876700024673280/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789762428.279207:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789762428.279245:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762428/real 1789762428] req@ffffa06c1872c700 x1876700024673024/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762453 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762428.279268:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872c700 x1876700024673024/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762453 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762428.279278:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762428.279281:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789762428.279284:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024673024:192.168.203.107@tcp:250:kworker.0 00000100:00100000:2.0:1789762428.279303:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024673152:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789762428.279320:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789762428.279330:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762428/real 1789762428] req@ffffa06c1872dc00 x1876700024673152/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762453 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762428.279339:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872dc00 x1876700024673152/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762453 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762428.279345:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762428.279347:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789762428.279350:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024673152:192.168.203.107@tcp:38:kworker.0 00000100:00100000:2.0:1789762428.279353:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024673280:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789762428.279364:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789762428.279371:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762428/real 1789762428] req@ffffa06c1872ed80 x1876700024673280/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762453 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762428.279379:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872ed80 x1876700024673280/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762453 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762428.279395:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762428.279397:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789762428.279400:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024673280:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789762428.535014:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762428.790969:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789762429.559095:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762429.814956:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762430.583026:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762430.839029:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762431.607007:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762431.862942:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789762432.631094:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762432.887012:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:1.0:1789762433.399007:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762433.399022:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789762433.399029:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762433.399068:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1867 00000100:00080000:1.0:1789762433.399086:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762433.399097:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789762433.399171:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2d500 x1876700024673408/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762433.399220:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762433.399226:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762433.399228:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762433.399234:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1867 00000100:00080000:1.0:1789762433.399239:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762433.399243:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789762433.399259:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762433.399281:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2ed80 x1876700024673536/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762433.399299:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762433.399304:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762433.399307:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762433.399314:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1867 00000100:00080000:1.0:1789762433.399318:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762433.399331:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789762433.399333:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024673408:192.168.203.107@tcp:250:kworker.0 00000008:00100000:1.0:1789762433.399369:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762433.399390:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2e300 x1876700024673664/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789762433.399443:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789762433.399555:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762433/real 1789762433] req@ffffa06c08e2d500 x1876700024673408/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762458 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762433.399581:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2d500 x1876700024673408/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762458 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762433.399597:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762433.399604:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789762433.399609:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024673408:192.168.203.107@tcp:250:kworker.0 00000100:00100000:2.0:1789762433.399620:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024673536:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789762433.399653:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789762433.399668:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762433/real 1789762433] req@ffffa06c08e2ed80 x1876700024673536/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762458 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762433.399680:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2ed80 x1876700024673536/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762458 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762433.399688:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762433.399691:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789762433.399711:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024673536:192.168.203.107@tcp:38:kworker.0 00000100:00100000:2.0:1789762433.399737:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024673664:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789762433.399759:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789762433.399769:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762433/real 1789762433] req@ffffa06c08e2e300 x1876700024673664/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762458 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762433.399779:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2e300 x1876700024673664/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762458 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762433.399805:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762433.399809:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789762433.399813:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024673664:192.168.203.107@tcp:38:kworker.0 00010000:00010000:2.0:1789762433.655261:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762433.911084:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762434.679036:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762434.934989:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762435.703055:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762435.958981:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762436.727085:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762436.983022:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762437.751052:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762438.006979:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789762438.518950:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762438.518958:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789762438.518962:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762438.518969:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1872 00000100:00080000:0.0:1789762438.518979:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762438.518984:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789762438.519018:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d1f80 x1876700024673792/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762438.519048:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762438.519051:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762438.519052:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762438.519071:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1872 00000100:00080000:0.0:1789762438.519075:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762438.519079:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789762438.519087:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762438.519098:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0700 x1876700024673920/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762438.519110:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762438.519123:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762438.519124:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762438.519128:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1872 00000100:00080000:0.0:1789762438.519132:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762438.519138:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789762438.519138:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024673792:192.168.203.107@tcp:250:kworker.0 00000008:00100000:0.0:1789762438.519153:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762438.519162:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0a80 x1876700024674048/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789762438.519249:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789762438.519303:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762438/real 1789762438] req@ffffa06c091d1f80 x1876700024673792/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762463 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762438.519323:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d1f80 x1876700024673792/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762463 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762438.519353:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762438.519358:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789762438.519362:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024673792:192.168.203.107@tcp:250:kworker.0 00000100:00100000:2.0:1789762438.519404:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024673920:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789762438.519448:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789762438.519464:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762438/real 1789762438] req@ffffa06c091d0700 x1876700024673920/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762463 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762438.519500:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0700 x1876700024673920/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762463 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762438.519507:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762438.519509:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789762438.519512:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024673920:192.168.203.107@tcp:38:kworker.0 00000100:00100000:2.0:1789762438.519517:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024674048:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789762438.519532:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789762438.519539:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762438/real 1789762438] req@ffffa06c091d0a80 x1876700024674048/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762463 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762438.519563:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0a80 x1876700024674048/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762463 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762438.519571:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762438.519573:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789762438.519576:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024674048:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789762438.775060:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762439.030998:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762439.799071:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762440.055044:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789762440.823153:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762441.078973:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762441.847184:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762442.102949:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762442.871063:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762443.126966:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789762443.638948:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762443.638965:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789762443.638969:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762443.638982:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1877 00000100:00080000:0.0:1789762443.638998:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762443.639009:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789762443.639062:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d3100 x1876700024674176/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762443.639091:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762443.639113:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762443.639119:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762443.639123:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1877 00000100:00080000:0.0:1789762443.639127:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762443.639131:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789762443.639140:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762443.639152:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0380 x1876700024674304/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762443.639163:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762443.639167:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:2.0:1789762443.639170:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024674176:192.168.203.107@tcp:250:kworker.0 00000100:00080000:0.0:1789762443.639171:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762443.639178:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1877 00000100:00080000:0.0:1789762443.639181:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762443.639183:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789762443.639192:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762443.639202:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d2680 x1876700024674432/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789762443.639253:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789762443.639319:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762443/real 1789762443] req@ffffa06c091d3100 x1876700024674176/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762468 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762443.639341:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d3100 x1876700024674176/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762468 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762443.639354:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762443.639358:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789762443.639363:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024674176:192.168.203.107@tcp:250:kworker.0 00000100:00100000:2.0:1789762443.639430:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024674304:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789762443.639465:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789762443.639475:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762443/real 1789762443] req@ffffa06c091d0380 x1876700024674304/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762468 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762443.639484:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0380 x1876700024674304/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762468 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762443.639490:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762443.639493:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789762443.639495:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024674304:192.168.203.107@tcp:38:kworker.0 00000100:00100000:2.0:1789762443.639506:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024674432:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789762443.639543:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789762443.639553:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762443/real 1789762443] req@ffffa06c091d2680 x1876700024674432/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762468 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762443.639564:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d2680 x1876700024674432/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762468 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762443.639571:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762443.639574:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789762443.639577:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024674432:192.168.203.107@tcp:38:kworker.0 00010000:00010000:1.0:1789762443.895130:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762444.150956:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762444.918975:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762445.175033:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762445.943114:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762446.199080:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762446.967142:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762447.222992:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762447.991078:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762448.247002:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789762448.758987:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762448.759004:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789762448.759011:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762448.759028:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1882 00000100:00080000:0.0:1789762448.759046:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762448.759058:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789762448.759121:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d1500 x1876700024674560/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762448.759192:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762448.759198:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762448.759201:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762448.759207:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1882 00000100:00080000:0.0:1789762448.759213:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762448.759217:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789762448.759233:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762448.759252:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d2a00 x1876700024674688/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762448.759281:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1789762448.759291:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024674560:192.168.203.107@tcp:250:kworker.0 00000100:00080000:0.0:1789762448.759292:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762448.759297:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762448.759309:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1882 00000100:00080000:0.0:1789762448.759317:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762448.759322:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789762448.759328:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762448.759344:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0e00 x1876700024674816/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789762448.759467:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789762448.759553:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762448/real 1789762448] req@ffffa06c091d1500 x1876700024674560/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762473 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762448.759582:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d1500 x1876700024674560/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762473 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762448.759597:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762448.759615:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789762448.759621:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024674560:192.168.203.107@tcp:250:kworker.0 00000100:00100000:2.0:1789762448.759661:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024674688:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789762448.759696:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789762448.759712:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762448/real 1789762448] req@ffffa06c091d2a00 x1876700024674688/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762473 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762448.759729:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d2a00 x1876700024674688/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762473 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762448.759740:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762448.759744:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789762448.759749:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024674688:192.168.203.107@tcp:38:kworker.0 00000100:00100000:2.0:1789762448.759756:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024674816:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789762448.759776:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789762448.759787:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762448/real 1789762448] req@ffffa06c091d0e00 x1876700024674816/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762473 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762448.759801:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0e00 x1876700024674816/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762473 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762448.759812:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762448.759815:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789762448.759819:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024674816:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789762449.015033:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762449.270913:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762450.039166:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762450.294996:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762451.063019:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762451.318950:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762452.087130:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762452.342998:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789762453.111160:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762453.366936:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789762453.879001:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762453.879014:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789762453.879019:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762453.879048:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1887 00000100:00080000:0.0:1789762453.879090:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762453.879099:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789762453.879149:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0000 x1876700024674944/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762453.879180:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762453.879184:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762453.879186:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762453.879190:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1887 00000100:00080000:0.0:1789762453.879193:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762453.879196:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789762453.879207:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762453.879218:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d3b80 x1876700024675072/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789762453.879231:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024674944:192.168.203.107@tcp:250:kworker.0 00000100:00080000:0.0:1789762453.879232:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762453.879241:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762453.879245:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762453.879249:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1887 00000100:00080000:0.0:1789762453.879252:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762453.879256:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789762453.879259:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762453.879271:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d2300 x1876700024675200/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789762453.879362:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789762453.879420:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762453/real 1789762453] req@ffffa06c091d0000 x1876700024674944/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762478 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762453.879447:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0000 x1876700024674944/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762478 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762453.879463:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762453.879469:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789762453.879475:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024674944:192.168.203.107@tcp:250:kworker.0 00000100:00100000:2.0:1789762453.879503:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024675072:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789762453.879527:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789762453.879539:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762453/real 1789762453] req@ffffa06c091d3b80 x1876700024675072/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762478 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762453.879550:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d3b80 x1876700024675072/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762478 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762453.879558:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762453.879561:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789762453.879563:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024675072:192.168.203.107@tcp:38:kworker.0 00000100:00100000:2.0:1789762453.879577:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024675200:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789762453.879595:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789762453.879621:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762453/real 1789762453] req@ffffa06c091d2300 x1876700024675200/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762478 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762453.879632:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d2300 x1876700024675200/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762478 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762453.879639:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762453.879642:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789762453.879644:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024675200:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789762454.135004:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762454.390948:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762455.159056:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762455.414938:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762456.182993:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762456.439006:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762457.207015:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762457.462960:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762458.231009:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762458.486927:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789762458.998983:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762458.998992:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789762458.998995:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762458.999003:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1892 00000100:00080000:2.0:1789762458.999012:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762458.999019:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789762458.999076:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e52a00 x1876700024675328/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762458.999117:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762458.999146:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762458.999150:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762458.999159:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1892 00000100:00080000:2.0:1789762458.999167:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762458.999173:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789762458.999185:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762458.999188:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e52a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024675328:192.168.203.107@tcp:250:kworker.0 00000100:00100000:2.0:1789762458.999209:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e53100 x1876700024675456/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762458.999227:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762458.999233:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762458.999236:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762458.999246:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1892 00000100:00080000:2.0:1789762458.999259:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762458.999267:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000800:00000100:3.0:1789762458.999294:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000008:00100000:2.0:1789762458.999296:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762458.999329:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e52d80 x1876700024675584/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789762458.999390:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762458/real 1789762458] req@ffffa06c07e52a00 x1876700024675328/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762483 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762458.999424:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e52a00 x1876700024675328/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762483 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762458.999439:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762458.999446:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789762458.999456:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e52a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024675328:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789762458.999489:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e53100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024675456:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789762458.999517:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789762458.999548:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762458/real 1789762458] req@ffffa06c07e53100 x1876700024675456/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762483 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762458.999572:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e53100 x1876700024675456/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762483 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762458.999581:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762458.999585:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789762458.999685:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e53100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024675456:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789762458.999693:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e52d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024675584:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789762458.999717:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789762458.999951:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762458/real 1789762458] req@ffffa06c07e52d80 x1876700024675584/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762483 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762458.999962:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e52d80 x1876700024675584/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762483 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762458.999971:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762458.999974:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789762458.999978:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e52d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024675584:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789762459.255016:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762459.510962:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762460.279092:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762460.534971:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762461.302969:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762461.558969:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762462.326983:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762462.583021:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762463.351031:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762463.606974:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:1.0:1789762464.118944:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762464.118952:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789762464.118956:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762464.118965:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1897 00000100:00080000:1.0:1789762464.118974:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762464.118981:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789762464.119014:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2c000 x1876700024675712/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762464.119042:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762464.119047:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762464.119049:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762464.119054:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1897 00000100:00080000:1.0:1789762464.119076:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762464.119082:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789762464.119092:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762464.119113:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2c700 x1876700024675840/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789762464.119113:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024675712:192.168.203.107@tcp:250:kworker.0 00000100:00080000:1.0:1789762464.119156:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762464.119173:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762464.119175:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762464.119180:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1897 00000100:00080000:1.0:1789762464.119184:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762464.119188:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789762464.119193:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762464.119206:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2ca80 x1876700024675968/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789762464.119250:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789762464.119322:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762464/real 1789762464] req@ffffa06c08e2c000 x1876700024675712/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762489 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762464.119338:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2c000 x1876700024675712/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762489 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762464.119359:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762464.119363:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789762464.119376:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024675712:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789762464.119404:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024675840:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789762464.119424:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789762464.119434:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762464/real 1789762464] req@ffffa06c08e2c700 x1876700024675840/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762489 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762464.119443:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2c700 x1876700024675840/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762489 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762464.119449:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762464.119451:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789762464.119454:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024675840:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789762464.119457:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024675968:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789762464.119469:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789762464.119475:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762464/real 1789762464] req@ffffa06c08e2ca80 x1876700024675968/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762489 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762464.119482:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2ca80 x1876700024675968/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762489 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762464.119488:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762464.119490:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789762464.119491:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024675968:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789762464.374994:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762464.630985:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762465.399042:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762465.655016:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789762466.422992:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762466.678936:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762467.446997:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762467.702973:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762468.471035:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762468.726951:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:1.0:1789762469.238990:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762469.239001:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789762469.239006:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762469.239018:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1903 00000100:00080000:1.0:1789762469.239030:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762469.239037:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789762469.239081:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2d180 x1876700024676096/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762469.239108:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762469.239112:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762469.239114:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762469.239119:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1903 00000100:00080000:1.0:1789762469.239123:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762469.239127:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789762469.239139:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762469.239176:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024676096:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789762469.239179:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2ca80 x1876700024676224/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762469.239194:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762469.239214:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762469.239216:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762469.239219:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1903 00000100:00080000:1.0:1789762469.239223:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762469.239226:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789762469.239231:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762469.239242:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2c700 x1876700024676352/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789762469.239286:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789762469.239339:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762469/real 1789762469] req@ffffa06c08e2d180 x1876700024676096/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762494 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762469.239357:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2d180 x1876700024676096/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762494 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762469.239367:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762469.239370:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789762469.239373:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024676096:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789762469.239397:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024676224:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789762469.239414:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789762469.239423:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762469/real 1789762469] req@ffffa06c08e2ca80 x1876700024676224/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762494 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762469.239431:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2ca80 x1876700024676224/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762494 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762469.239437:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762469.239440:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789762469.239442:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024676224:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789762469.239446:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024676352:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789762469.239457:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789762469.239473:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762469/real 1789762469] req@ffffa06c08e2c700 x1876700024676352/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762494 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762469.239486:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2c700 x1876700024676352/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762494 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762469.239495:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762469.239499:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789762469.239503:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024676352:192.168.203.107@tcp:38:kworker.0 00010000:00010000:1.0:1789762469.495037:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762469.750966:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762470.518994:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762470.774925:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762471.542971:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762471.798975:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789762472.566965:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762472.822948:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789762473.590961:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762473.846920:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789762474.358989:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762474.358997:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789762474.359001:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762474.359010:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1908 00000100:00080000:0.0:1789762474.359019:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762474.359025:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789762474.359072:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d2300 x1876700024676480/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762474.359096:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762474.359099:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762474.359100:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762474.359104:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1908 00000100:00080000:0.0:1789762474.359106:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762474.359109:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789762474.359134:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762474.359144:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d3b80 x1876700024676608/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762474.359154:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1789762474.359157:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024676480:192.168.203.107@tcp:250:kworker.0 00000100:00080000:0.0:1789762474.359199:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762474.359204:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762474.359209:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1908 00000100:00080000:0.0:1789762474.359215:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762474.359220:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789762474.359226:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1789762474.359255:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789762474.359259:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0000 x1876700024676736/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789762474.359299:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762474/real 1789762474] req@ffffa06c091d2300 x1876700024676480/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762499 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762474.359324:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d2300 x1876700024676480/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762499 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762474.359332:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762474.359335:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789762474.359338:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024676480:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789762474.359356:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024676608:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789762474.359371:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789762474.359379:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762474/real 1789762474] req@ffffa06c091d3b80 x1876700024676608/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762499 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762474.359400:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d3b80 x1876700024676608/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762499 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762474.359406:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762474.359408:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789762474.359410:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024676608:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789762474.359414:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024676736:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789762474.359428:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789762474.359434:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762474/real 1789762474] req@ffffa06c091d0000 x1876700024676736/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762499 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762474.359442:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0000 x1876700024676736/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762499 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762474.359447:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762474.359449:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789762474.359451:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024676736:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789762474.615057:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762474.870939:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789762475.639012:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762475.894928:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789762476.662953:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762476.918925:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762477.686954:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762477.942941:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789762478.711010:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762478.966937:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:1.0:1789762479.478972:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762479.478982:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789762479.478986:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762479.478996:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1913 00000100:00080000:1.0:1789762479.479007:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762479.479014:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789762479.479051:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2c000 x1876700024676864/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762479.479091:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762479.479094:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762479.479096:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762479.479099:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1913 00000100:00080000:1.0:1789762479.479102:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762479.479105:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789762479.479112:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762479.479138:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2e300 x1876700024676992/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762479.479151:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762479.479155:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762479.479157:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1789762479.479161:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024676864:192.168.203.107@tcp:250:kworker.0 00000100:00080000:1.0:1789762479.479162:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1913 00000100:00080000:1.0:1789762479.479168:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762479.479175:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789762479.479181:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762479.479195:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2ed80 x1876700024677120/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789762479.479232:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789762479.479275:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762479/real 1789762479] req@ffffa06c08e2c000 x1876700024676864/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762504 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762479.479293:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2c000 x1876700024676864/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762504 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762479.479311:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762479.479316:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789762479.479320:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024676864:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789762479.479344:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024676992:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789762479.479370:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789762479.479383:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762479/real 1789762479] req@ffffa06c08e2e300 x1876700024676992/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762504 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762479.479396:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2e300 x1876700024676992/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762504 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762479.479405:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762479.479409:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789762479.479412:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024676992:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789762479.479418:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024677120:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789762479.479438:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789762479.479449:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762479/real 1789762479] req@ffffa06c08e2ed80 x1876700024677120/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762504 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762479.479461:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2ed80 x1876700024677120/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762504 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762479.479470:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762479.479474:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789762479.479477:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024677120:192.168.203.107@tcp:38:kworker.0 00010000:00010000:0.0:1789762479.734969:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762479.990944:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762480.758978:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762481.014919:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762481.782992:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762482.038920:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789762482.806980:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762483.062927:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762483.831007:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762484.086912:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789762484.599961:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762484.599972:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789762484.599976:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762484.599988:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1918 00000100:00080000:2.0:1789762484.600000:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762484.600008:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789762484.600052:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e52d80 x1876700024677248/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762484.600093:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762484.600098:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762484.600100:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762484.600105:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1918 00000100:00080000:2.0:1789762484.600109:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762484.600113:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789762484.600124:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762484.600145:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e52d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024677248:192.168.203.107@tcp:250:kworker.0 00000100:00100000:2.0:1789762484.600157:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e53100 x1876700024677376/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762484.600170:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762484.600173:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762484.600174:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762484.600178:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1918 00000100:00080000:2.0:1789762484.600181:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762484.600184:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789762484.600189:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762484.600197:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e52a00 x1876700024677504/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789762484.600270:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e53100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024677376:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789762484.600302:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e52a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024677504:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789762484.855022:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762485.110909:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789762485.879058:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762486.135022:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762486.902997:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762487.158971:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000400:00000100:3.0:1789762487.672108:0:2331:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.107:988 00000400:00000100:3.0:1789762487.672192:0:2331:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.107@tcp at host 192.168.203.107 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1789762487.672217:0:2331:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.107@tcp 00000800:00000100:3.0:1789762487.672288:0:2331:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.107@tcp 00000800:00000100:3.0:1789762487.672310:0:2331:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.107@tcp 00000100:00100000:2.0:1789762487.672317:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762484/real 1789762487] req@ffffa06c07e52d80 x1876700024677248/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762509 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762487.672348:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e52d80 x1876700024677248/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762509 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762487.672357:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762487.672360:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789762487.672363:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e52d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024677248:192.168.203.107@tcp:250:kworker.0 00000100:00100000:2.0:1789762487.672382:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762484/real 1789762487] req@ffffa06c07e53100 x1876700024677376/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762509 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762487.672390:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e53100 x1876700024677376/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762509 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762487.672415:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762487.672419:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789762487.672423:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e53100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024677376:192.168.203.107@tcp:38:kworker.0 00000100:00100000:2.0:1789762487.672431:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762484/real 1789762487] req@ffffa06c07e52a00 x1876700024677504/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762509 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762487.672445:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e52a00 x1876700024677504/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762509 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762487.672453:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762487.672456:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789762487.672459:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e52a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024677504:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789762487.927029:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762488.182938:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789762488.950943:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762489.206961:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789762489.718929:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762489.718937:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789762489.718940:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762489.718949:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1923 00000100:00080000:0.0:1789762489.718959:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762489.718967:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789762489.719003:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0e00 x1876700024677632/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762489.719029:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762489.719032:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762489.719033:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762489.719036:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1923 00000100:00080000:0.0:1789762489.719039:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762489.719042:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789762489.719061:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762489.719077:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d2a00 x1876700024677760/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789762489.719089:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024677632:192.168.203.107@tcp:250:kworker.0 00000100:00080000:0.0:1789762489.719092:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762489.719101:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762489.719121:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762489.719131:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1923 00000100:00080000:0.0:1789762489.719136:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762489.719142:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789762489.719148:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762489.719165:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d1500 x1876700024677888/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789762489.719166:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789762489.719225:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762489/real 1789762489] req@ffffa06c091d0e00 x1876700024677632/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762514 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762489.719257:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0e00 x1876700024677632/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762514 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762489.719270:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762489.719274:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789762489.719279:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024677632:192.168.203.107@tcp:250:kworker.0 00000100:00100000:2.0:1789762489.719308:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024677760:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789762489.719344:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789762489.719358:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762489/real 1789762489] req@ffffa06c091d2a00 x1876700024677760/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762514 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762489.719371:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d2a00 x1876700024677760/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762514 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762489.719379:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762489.719382:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789762489.719386:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024677760:192.168.203.107@tcp:38:kworker.0 00000100:00100000:2.0:1789762489.719392:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024677888:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789762489.719412:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789762489.719424:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762489/real 1789762489] req@ffffa06c091d1500 x1876700024677888/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762514 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762489.719437:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d1500 x1876700024677888/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762514 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762489.719445:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762489.719449:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789762489.719452:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024677888:192.168.203.107@tcp:38:kworker.0 00010000:00010000:2.0:1789762489.974942:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762490.230921:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762490.998960:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762491.254923:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789762492.022963:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762492.278917:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789762493.047010:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762493.302920:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762494.070946:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762494.326904:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789762494.838928:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762494.838936:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789762494.838939:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762494.838949:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1928 00000100:00080000:0.0:1789762494.838958:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762494.838965:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789762494.838996:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d2680 x1876700024678016/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762494.839035:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762494.839038:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762494.839040:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762494.839044:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1928 00000100:00080000:0.0:1789762494.839047:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762494.839050:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789762494.839058:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762494.839072:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0380 x1876700024678144/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789762494.839095:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024678016:192.168.203.107@tcp:250:kworker.0 00000100:00080000:0.0:1789762494.839097:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762494.839105:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762494.839107:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762494.839111:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1928 00000100:00080000:0.0:1789762494.839114:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762494.839118:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789762494.839123:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762494.839136:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d3100 x1876700024678272/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789762494.839168:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762494.839200:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762494/real 1789762494] req@ffffa06c091d2680 x1876700024678016/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762519 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762494.839213:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d2680 x1876700024678016/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762519 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762494.839222:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762494.839225:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762494.839229:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024678016:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789762494.839250:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024678144:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789762494.839270:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762494.839280:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762494/real 1789762494] req@ffffa06c091d0380 x1876700024678144/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762519 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762494.839300:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0380 x1876700024678144/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762519 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762494.839306:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762494.839308:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762494.839311:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024678144:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789762494.839314:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024678272:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789762494.839330:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762494.839337:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762494/real 1789762494] req@ffffa06c091d3100 x1876700024678272/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762519 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762494.839346:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d3100 x1876700024678272/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762519 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762494.839351:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762494.839354:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762494.839356:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024678272:192.168.203.107@tcp:38:kworker.0 00010000:00010000:1.0:1789762495.094960:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762495.350949:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789762496.119031:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762496.374919:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762497.143026:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762497.398940:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789762498.167043:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762498.422939:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762499.191017:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762499.446943:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789762499.959017:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762499.959029:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789762499.959034:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762499.959046:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1933 00000100:00080000:0.0:1789762499.959058:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762499.959067:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789762499.959109:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0a80 x1876700024678400/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762499.959138:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762499.959142:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762499.959145:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762499.959150:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1933 00000100:00080000:0.0:1789762499.959155:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762499.959179:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789762499.959192:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762499.959193:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024678400:192.168.203.107@tcp:250:kworker.0 00000100:00100000:0.0:1789762499.959225:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0700 x1876700024678528/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762499.959245:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762499.959250:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762499.959253:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762499.959258:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1933 00000100:00080000:0.0:1789762499.959264:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762499.959270:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789762499.959278:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762499.959294:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d1f80 x1876700024678656/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789762499.959301:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762499.959359:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762499/real 1789762499] req@ffffa06c091d0a80 x1876700024678400/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762524 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762499.959389:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0a80 x1876700024678400/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762524 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762499.959407:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762499.959415:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762499.959422:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024678400:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789762499.959453:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024678528:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789762499.959515:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762499.959540:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762499/real 1789762499] req@ffffa06c091d0700 x1876700024678528/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762524 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762499.959563:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0700 x1876700024678528/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762524 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762499.959574:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762499.959577:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762499.959581:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024678528:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789762499.959589:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024678656:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789762499.959615:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762499.959627:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762499/real 1789762499] req@ffffa06c091d1f80 x1876700024678656/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762524 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762499.959659:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d1f80 x1876700024678656/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762524 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762499.959668:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762499.959672:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762499.959676:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024678656:192.168.203.107@tcp:38:kworker.0 00010000:00010000:2.0:1789762500.215092:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762500.470915:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762501.239050:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762501.494950:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762502.263093:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762502.519073:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762503.287033:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762503.542989:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762504.311038:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762504.566917:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:1.0:1789762505.078974:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762505.078985:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789762505.078989:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762505.078999:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1938 00000100:00080000:1.0:1789762505.079008:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762505.079014:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789762505.079045:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2d500 x1876700024678784/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762505.079069:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762505.079072:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762505.079073:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762505.079077:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1938 00000100:00080000:1.0:1789762505.079080:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762505.079096:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789762505.079101:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024678784:192.168.203.107@tcp:250:kworker.0 00000008:00100000:1.0:1789762505.079106:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762505.079120:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2ed80 x1876700024678912/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762505.079130:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762505.079132:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762505.079134:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762505.079137:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1938 00000100:00080000:1.0:1789762505.079140:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762505.079142:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789762505.079146:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762505.079154:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2e300 x1876700024679040/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789762505.079165:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789762505.079199:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762505/real 1789762505] req@ffffa06c08e2d500 x1876700024678784/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762530 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762505.079213:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2d500 x1876700024678784/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762530 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762505.079220:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762505.079223:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789762505.079226:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024678784:192.168.203.107@tcp:250:kworker.0 00000100:00100000:2.0:1789762505.079244:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024678912:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789762505.079259:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789762505.079267:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762505/real 1789762505] req@ffffa06c08e2ed80 x1876700024678912/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762530 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762505.079275:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2ed80 x1876700024678912/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762530 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762505.079280:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762505.079283:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789762505.079285:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024678912:192.168.203.107@tcp:38:kworker.0 00000100:00100000:2.0:1789762505.079288:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024679040:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789762505.079299:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789762505.079304:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762505/real 1789762505] req@ffffa06c08e2e300 x1876700024679040/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762530 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762505.079330:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2e300 x1876700024679040/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762530 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762505.079336:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762505.079338:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789762505.079340:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024679040:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789762505.334963:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762505.590913:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789762506.359071:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762506.614964:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762507.383075:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762507.638938:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762508.406975:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762508.662977:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789762509.430977:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762509.686912:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789762510.198953:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762510.198961:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789762510.198964:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762510.198973:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1944 00000100:00080000:2.0:1789762510.198982:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762510.198987:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789762510.199017:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e52a00 x1876700024679168/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762510.199039:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762510.199042:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762510.199044:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762510.199047:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1944 00000100:00080000:2.0:1789762510.199049:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762510.199052:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789762510.199059:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762510.199069:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e53100 x1876700024679296/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789762510.199074:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e52a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024679168:192.168.203.107@tcp:250:kworker.0 00000100:00080000:2.0:1789762510.199099:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762510.199102:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762510.199104:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762510.199108:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1944 00000100:00080000:2.0:1789762510.199112:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762510.199115:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789762510.199118:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762510.199128:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e52d80 x1876700024679424/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789762510.199132:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762510.199169:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762510/real 1789762510] req@ffffa06c07e52a00 x1876700024679168/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762535 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762510.199182:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e52a00 x1876700024679168/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762535 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762510.199196:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762510.199199:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762510.199202:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e52a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024679168:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789762510.199220:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e53100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024679296:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789762510.199236:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762510.199244:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762510/real 1789762510] req@ffffa06c07e53100 x1876700024679296/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762535 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762510.199252:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e53100 x1876700024679296/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762535 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762510.199258:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762510.199260:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762510.199262:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e53100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024679296:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789762510.199265:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e52d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024679424:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789762510.199275:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762510.199281:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762510/real 1789762510] req@ffffa06c07e52d80 x1876700024679424/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762535 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762510.199288:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e52d80 x1876700024679424/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762535 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762510.199293:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762510.199295:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762510.199297:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e52d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024679424:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789762510.454955:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762510.710953:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762511.479009:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762511.734911:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762512.503033:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762512.758990:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762513.527025:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762513.782933:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762514.551006:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762514.806947:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:1.0:1789762515.318951:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762515.318959:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789762515.318963:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762515.318971:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1949 00000100:00080000:1.0:1789762515.318981:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762515.318986:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789762515.319019:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2c000 x1876700024679552/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762515.319044:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762515.319047:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762515.319048:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762515.319051:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1949 00000100:00080000:1.0:1789762515.319054:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762515.319056:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789762515.319064:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762515.319074:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2c700 x1876700024679680/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789762515.319077:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024679552:192.168.203.107@tcp:250:kworker.0 00000100:00080000:1.0:1789762515.319091:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762515.319093:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762515.319108:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762515.319112:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1949 00000100:00080000:1.0:1789762515.319115:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762515.319118:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789762515.319122:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762515.319131:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2ca80 x1876700024679808/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789762515.319147:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789762515.319179:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762515/real 1789762515] req@ffffa06c08e2c000 x1876700024679552/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762540 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762515.319193:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2c000 x1876700024679552/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762540 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762515.319201:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762515.319204:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789762515.319207:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024679552:192.168.203.107@tcp:250:kworker.0 00000100:00100000:2.0:1789762515.319224:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024679680:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789762515.319239:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789762515.319254:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762515/real 1789762515] req@ffffa06c08e2c700 x1876700024679680/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762540 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762515.319263:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2c700 x1876700024679680/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762540 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762515.319269:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762515.319271:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789762515.319273:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024679680:192.168.203.107@tcp:38:kworker.0 00000100:00100000:2.0:1789762515.319277:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024679808:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789762515.319289:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789762515.319295:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762515/real 1789762515] req@ffffa06c08e2ca80 x1876700024679808/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762540 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762515.319303:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2ca80 x1876700024679808/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762540 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762515.319308:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762515.319310:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789762515.319312:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024679808:192.168.203.107@tcp:38:kworker.0 00010000:00010000:2.0:1789762515.574973:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762515.830935:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762516.598957:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762516.854963:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762517.622967:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762517.878905:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762518.647049:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762518.902995:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762519.671032:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762519.926904:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789762520.438931:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762520.438939:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789762520.438943:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762520.438951:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1954 00000100:00080000:2.0:1789762520.438960:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762520.438982:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789762520.439010:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50700 x1876700024679936/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762520.439032:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762520.439035:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762520.439036:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762520.439040:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1954 00000100:00080000:2.0:1789762520.439043:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762520.439045:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789762520.439052:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762520.439061:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50a80 x1876700024680064/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762520.439070:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762520.439073:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762520.439075:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1789762520.439079:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024679936:192.168.203.107@tcp:250:kworker.0 00000100:00080000:2.0:1789762520.439080:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1954 00000100:00080000:2.0:1789762520.439084:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762520.439090:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789762520.439093:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762520.439101:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e53480 x1876700024680192/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789762520.439141:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762520.439174:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762520/real 1789762520] req@ffffa06c07e50700 x1876700024679936/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762545 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762520.439204:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50700 x1876700024679936/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762545 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762520.439214:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762520.439218:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762520.439221:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024679936:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789762520.439243:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024680064:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789762520.439298:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762520.439312:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762520/real 1789762520] req@ffffa06c07e50a80 x1876700024680064/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762545 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762520.439346:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50a80 x1876700024680064/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762545 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762520.439356:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762520.439359:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762520.439363:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024680064:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789762520.439370:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e53480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024680192:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789762520.439391:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762520.439400:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762520/real 1789762520] req@ffffa06c07e53480 x1876700024680192/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762545 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762520.439409:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e53480 x1876700024680192/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762545 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762520.439414:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762520.439416:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762520.439418:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e53480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024680192:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789762520.694982:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762520.950949:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762521.719067:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762521.974912:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762522.742970:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762522.998951:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789762523.766980:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762524.022892:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762524.790953:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762525.046904:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789762525.558940:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762525.558949:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789762525.558967:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762525.558976:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1959 00000100:00080000:2.0:1789762525.558984:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762525.558989:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789762525.559017:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51c00 x1876700024680320/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762525.559040:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762525.559043:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762525.559045:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762525.559047:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1959 00000100:00080000:2.0:1789762525.559050:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762525.559052:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789762525.559059:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762525.559068:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e53800 x1876700024680448/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762525.559077:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762525.559080:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762525.559081:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1789762525.559087:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024680320:192.168.203.107@tcp:250:kworker.0 00000100:00080000:2.0:1789762525.559087:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1959 00000100:00080000:2.0:1789762525.559092:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762525.559097:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789762525.559101:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762525.559110:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e52680 x1876700024680576/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789762525.559145:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762525.559178:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762525/real 1789762525] req@ffffa06c07e51c00 x1876700024680320/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762550 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762525.559198:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51c00 x1876700024680320/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762550 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762525.559206:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762525.559209:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762525.559211:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024680320:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789762525.559229:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e53800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024680448:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789762525.559248:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762525.559262:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762525/real 1789762525] req@ffffa06c07e53800 x1876700024680448/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762550 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762525.559275:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e53800 x1876700024680448/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762550 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762525.559285:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762525.559289:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762525.559293:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e53800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024680448:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789762525.559299:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e52680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024680576:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789762525.559320:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762525.559333:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762525/real 1789762525] req@ffffa06c07e52680 x1876700024680576/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762550 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762525.559345:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e52680 x1876700024680576/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762550 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762525.559367:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762525.559371:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762525.559374:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e52680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024680576:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789762525.814966:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762526.070921:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789762526.838989:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762527.094923:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762527.862997:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762528.118943:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762528.886947:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762529.142933:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789762529.911131:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762530.166946:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789762530.678917:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762530.678926:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789762530.678929:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762530.678938:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1964 00000100:00080000:2.0:1789762530.678946:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762530.678951:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789762530.678988:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51f80 x1876700024680704/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762530.679017:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762530.679021:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762530.679023:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762530.679027:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1964 00000100:00080000:2.0:1789762530.679030:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762530.679033:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789762530.679042:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762530.679057:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50380 x1876700024680832/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789762530.679058:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024680704:192.168.203.107@tcp:250:kworker.0 00000100:00080000:2.0:1789762530.679073:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762530.679077:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762530.679078:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762530.679081:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1964 00000100:00080000:2.0:1789762530.679084:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762530.679086:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789762530.679102:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762530.679110:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e53b80 x1876700024680960/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789762530.679119:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762530.679160:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762530/real 1789762530] req@ffffa06c07e51f80 x1876700024680704/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762555 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762530.679180:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51f80 x1876700024680704/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762555 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762530.679191:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762530.679195:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762530.679199:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024680704:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789762530.679224:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024680832:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789762530.679254:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762530.679272:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762530/real 1789762530] req@ffffa06c07e50380 x1876700024680832/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762555 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762530.679282:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50380 x1876700024680832/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762555 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762530.679300:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762530.679304:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762530.679308:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024680832:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789762530.679316:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e53b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024680960:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789762530.679344:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762530.679357:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762530/real 1789762530] req@ffffa06c07e53b80 x1876700024680960/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762555 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762530.679366:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e53b80 x1876700024680960/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762555 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762530.679373:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762530.679375:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762530.679379:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e53b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024680960:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789762530.934953:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762531.190928:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762531.959054:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762532.215026:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789762532.982951:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762533.238901:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762534.006969:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762534.262893:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762535.030947:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762535.286926:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789762535.798943:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762535.798952:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789762535.798956:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762535.798965:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1969 00000100:00080000:0.0:1789762535.798976:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762535.798982:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789762535.799015:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d1f80 x1876700024681088/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762535.799059:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762535.799064:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762535.799065:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762535.799069:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1969 00000100:00080000:0.0:1789762535.799073:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762535.799078:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789762535.799087:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762535.799101:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024681088:192.168.203.107@tcp:250:kworker.0 00000100:00100000:0.0:1789762535.799103:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0700 x1876700024681216/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762535.799118:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762535.799122:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762535.799124:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762535.799127:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1969 00000100:00080000:0.0:1789762535.799131:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762535.799135:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789762535.799140:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762535.799151:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0a80 x1876700024681344/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789762535.799160:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762535.799198:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762535/real 1789762535] req@ffffa06c091d1f80 x1876700024681088/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762560 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762535.799230:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d1f80 x1876700024681088/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762560 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762535.799242:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762535.799246:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762535.799251:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024681088:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789762535.799273:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024681216:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789762535.799293:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762535.799302:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762535/real 1789762535] req@ffffa06c091d0700 x1876700024681216/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762560 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762535.799311:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0700 x1876700024681216/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762560 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762535.799317:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762535.799319:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762535.799321:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024681216:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789762535.799327:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024681344:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789762535.799345:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762535.799357:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762535/real 1789762535] req@ffffa06c091d0a80 x1876700024681344/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762560 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762535.799371:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0a80 x1876700024681344/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762560 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762535.799389:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762535.799391:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762535.799394:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024681344:192.168.203.107@tcp:38:kworker.0 00010000:00010000:2.0:1789762536.054969:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762536.310955:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789762537.078983:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762537.334924:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762538.103057:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762538.358963:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789762539.126981:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762539.382935:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789762540.151059:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762540.406970:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789762540.918959:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762540.918967:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789762540.918971:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762540.918979:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1974 00000100:00080000:0.0:1789762540.919004:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762540.919010:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789762540.919039:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d3100 x1876700024681472/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762540.919063:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762540.919066:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762540.919067:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762540.919070:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1974 00000100:00080000:0.0:1789762540.919072:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762540.919074:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789762540.919082:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762540.919090:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0380 x1876700024681600/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762540.919098:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762540.919101:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762540.919102:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762540.919105:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1974 00000100:00080000:0.0:1789762540.919107:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762540.919110:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789762540.919113:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762540.919118:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024681472:192.168.203.107@tcp:250:kworker.0 00000100:00100000:0.0:1789762540.919122:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d2680 x1876700024681728/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789762540.919201:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762540.919254:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762540/real 1789762540] req@ffffa06c091d3100 x1876700024681472/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762565 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762540.919274:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d3100 x1876700024681472/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762565 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762540.919285:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762540.919290:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762540.919294:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024681472:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789762540.919303:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024681600:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789762540.919329:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762540.919344:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762540/real 1789762540] req@ffffa06c091d0380 x1876700024681600/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762565 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762540.919366:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0380 x1876700024681600/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762565 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762540.919375:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762540.919379:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762540.919383:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024681600:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789762540.919408:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024681728:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789762540.919434:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762540.919456:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762540/real 1789762540] req@ffffa06c091d2680 x1876700024681728/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762565 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762540.919471:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d2680 x1876700024681728/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762565 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762540.919480:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762540.919483:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762540.919486:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024681728:192.168.203.107@tcp:38:kworker.0 00010000:00010000:2.0:1789762541.175119:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762541.430899:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762542.198944:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762542.454919:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789762543.222944:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762543.478914:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762544.246955:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762544.502925:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762545.270969:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762545.526913:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:1.0:1789762546.038981:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762546.038996:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789762546.039016:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762546.039030:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1979 00000100:00080000:1.0:1789762546.039044:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762546.039053:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789762546.039123:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2d180 x1876700024681856/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762546.039164:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762546.039169:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762546.039171:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762546.039177:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1979 00000100:00080000:1.0:1789762546.039182:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762546.039187:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789762546.039201:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762546.039220:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2ca80 x1876700024681984/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762546.039234:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762546.039238:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762546.039241:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762546.039245:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1979 00000100:00080000:1.0:1789762546.039249:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762546.039253:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789762546.039260:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762546.039290:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2c700 x1876700024682112/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789762546.039330:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024681856:192.168.203.107@tcp:250:kworker.0 00000800:00000100:2.0:1789762546.039424:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789762546.039499:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762546/real 1789762546] req@ffffa06c08e2d180 x1876700024681856/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762571 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762546.039524:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2d180 x1876700024681856/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762571 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762546.039538:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762546.039544:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789762546.039549:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024681856:192.168.203.107@tcp:250:kworker.0 00000100:00100000:2.0:1789762546.039559:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024681984:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789762546.039585:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789762546.039599:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762546/real 1789762546] req@ffffa06c08e2ca80 x1876700024681984/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762571 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762546.039612:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2ca80 x1876700024681984/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762571 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762546.039666:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762546.039670:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789762546.039673:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024681984:192.168.203.107@tcp:38:kworker.0 00000100:00100000:2.0:1789762546.039698:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024682112:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789762546.039719:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789762546.039728:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762546/real 1789762546] req@ffffa06c08e2c700 x1876700024682112/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762571 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762546.039738:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2c700 x1876700024682112/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762571 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762546.039745:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762546.039748:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789762546.039750:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024682112:192.168.203.107@tcp:38:kworker.0 00010000:00010000:1.0:1789762546.294955:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762546.550932:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762547.318957:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762547.574933:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762548.342947:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762548.598928:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789762549.367031:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762549.622967:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762550.390991:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762550.646931:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789762551.158935:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762551.158955:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789762551.158958:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762551.158966:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1984 00000100:00080000:0.0:1789762551.158975:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762551.158980:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789762551.159009:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d1500 x1876700024682240/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762551.159029:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762551.159032:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762551.159033:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762551.159036:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1984 00000100:00080000:0.0:1789762551.159038:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762551.159040:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789762551.159047:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762551.159057:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d2a00 x1876700024682368/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762551.159065:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762551.159068:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762551.159069:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762551.159071:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1984 00000100:00080000:0.0:1789762551.159073:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1789762551.159077:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024682240:192.168.203.107@tcp:250:kworker.0 00000100:00080000:0.0:1789762551.159078:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789762551.159085:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762551.159092:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0e00 x1876700024682496/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789762551.159162:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024682368:192.168.203.107@tcp:38:kworker.0 00000100:00100000:2.0:1789762551.159194:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024682496:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789762551.414978:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762551.671003:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762552.438990:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762552.694927:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789762553.463017:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762553.718943:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000400:00000100:0.0:1789762554.231108:0:2328:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.107:988 00000400:00000100:0.0:1789762554.231156:0:2328:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.107@tcp at host 192.168.203.107 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1789762554.231174:0:2328:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.107@tcp 00000800:00000100:0.0:1789762554.231216:0:2328:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.107@tcp 00000800:00000100:0.0:1789762554.231223:0:2328:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.107@tcp 00000100:00100000:2.0:1789762554.231289:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762551/real 1789762554] req@ffffa06c091d0e00 x1876700024682496/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762576 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762554.231341:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0e00 x1876700024682496/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762576 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762554.231353:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762554.231371:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789762554.231376:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024682496:192.168.203.107@tcp:38:kworker.0 00000100:00100000:2.0:1789762554.231384:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762551/real 1789762554] req@ffffa06c091d1500 x1876700024682240/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762576 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762554.231397:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d1500 x1876700024682240/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762576 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762554.231405:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762554.231409:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789762554.231413:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024682240:192.168.203.107@tcp:250:kworker.0 00000100:00100000:2.0:1789762554.231421:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762551/real 1789762554] req@ffffa06c091d2a00 x1876700024682368/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762576 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762554.231434:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d2a00 x1876700024682368/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762576 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762554.231441:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762554.231444:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789762554.231448:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024682368:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789762554.486972:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762554.742914:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789762555.511066:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762555.766993:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789762556.278949:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762556.278965:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789762556.278968:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762556.278983:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1990 00000100:00080000:2.0:1789762556.278991:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762556.278996:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789762556.279025:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50e00 x1876700024682624/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762556.279047:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762556.279050:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762556.279051:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762556.279055:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1990 00000100:00080000:2.0:1789762556.279059:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762556.279063:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789762556.279073:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762556.279087:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51500 x1876700024682752/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789762556.279098:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024682624:192.168.203.107@tcp:250:kworker.0 00000100:00080000:2.0:1789762556.279104:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762556.279109:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762556.279111:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762556.279116:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1990 00000100:00080000:2.0:1789762556.279120:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762556.279124:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789762556.279128:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762556.279142:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50000 x1876700024682880/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789762556.279168:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762556.279229:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762556/real 1789762556] req@ffffa06c07e50e00 x1876700024682624/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762581 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762556.279248:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50e00 x1876700024682624/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762581 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762556.279259:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762556.279263:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762556.279268:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024682624:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789762556.279291:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024682752:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789762556.279318:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762556.279331:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762556/real 1789762556] req@ffffa06c07e51500 x1876700024682752/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762581 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762556.279361:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51500 x1876700024682752/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762581 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762556.279370:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762556.279374:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762556.279377:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024682752:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789762556.279384:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024682880:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789762556.279408:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762556.279421:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762556/real 1789762556] req@ffffa06c07e50000 x1876700024682880/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762581 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762556.279434:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50000 x1876700024682880/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762581 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762556.279442:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762556.279445:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762556.279448:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024682880:192.168.203.107@tcp:38:kworker.0 00010000:00010000:0.0:1789762556.534992:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762556.790933:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762557.558969:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762557.814905:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789762558.582976:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762558.838930:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762559.606959:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762559.862929:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789762560.630945:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762560.886896:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:1.0:1789762561.398993:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762561.399007:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789762561.399038:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762561.399051:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 1995 00000100:00080000:1.0:1789762561.399064:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762561.399074:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789762561.399118:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2c000 x1876700024683008/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762561.399159:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762561.399164:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762561.399167:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762561.399173:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 1995 00000100:00080000:1.0:1789762561.399178:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762561.399184:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789762561.399197:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762561.399214:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2e300 x1876700024683136/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789762561.399229:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024683008:192.168.203.107@tcp:250:kworker.0 00000100:00080000:1.0:1789762561.399236:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762561.399244:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762561.399246:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762561.399252:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 1995 00000100:00080000:1.0:1789762561.399257:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762561.399262:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789762561.399268:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762561.399285:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2ed80 x1876700024683264/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789762561.399334:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789762561.399391:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762561/real 1789762561] req@ffffa06c08e2c000 x1876700024683008/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762586 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762561.399415:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2c000 x1876700024683008/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762586 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762561.399429:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762561.399434:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789762561.399440:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024683008:192.168.203.107@tcp:250:kworker.0 00000100:00100000:2.0:1789762561.399468:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024683136:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789762561.399496:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789762561.399540:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762561/real 1789762561] req@ffffa06c08e2e300 x1876700024683136/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762586 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762561.399557:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2e300 x1876700024683136/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762586 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762561.399567:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762561.399572:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789762561.399576:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024683136:192.168.203.107@tcp:38:kworker.0 00000100:00100000:2.0:1789762561.399584:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024683264:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789762561.399608:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789762561.399639:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762561/real 1789762561] req@ffffa06c08e2ed80 x1876700024683264/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762586 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762561.399654:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2ed80 x1876700024683264/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762586 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762561.399664:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762561.399668:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789762561.399672:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024683264:192.168.203.107@tcp:38:kworker.0 00010000:00010000:1.0:1789762561.655105:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762561.910932:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762562.679090:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762562.934900:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789762563.702968:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762563.958924:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762564.727072:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762564.982979:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762565.750956:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762566.006957:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789762566.518946:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762566.518955:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789762566.518958:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762566.518967:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2000 00000100:00080000:2.0:1789762566.518976:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762566.518981:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789762566.519014:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51180 x1876700024683392/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762566.519038:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762566.519041:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762566.519043:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762566.519046:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2000 00000100:00080000:2.0:1789762566.519048:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762566.519050:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789762566.519057:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762566.519067:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e52300 x1876700024683520/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789762566.519093:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024683392:192.168.203.107@tcp:250:kworker.0 00000100:00080000:2.0:1789762566.519103:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762566.519109:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762566.519111:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762566.519116:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2000 00000100:00080000:2.0:1789762566.519121:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762566.519126:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789762566.519132:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762566.519147:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51880 x1876700024683648/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789762566.519169:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762566.519216:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762566/real 1789762566] req@ffffa06c07e51180 x1876700024683392/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762591 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762566.519238:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51180 x1876700024683392/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762591 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762566.519249:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762566.519254:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762566.519259:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024683392:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789762566.519284:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e52300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024683520:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789762566.519316:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762566.519334:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762566/real 1789762566] req@ffffa06c07e52300 x1876700024683520/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762591 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762566.519349:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e52300 x1876700024683520/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762591 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762566.519359:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762566.519363:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762566.519375:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e52300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024683520:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789762566.519384:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024683648:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789762566.519411:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762566.519427:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762566/real 1789762566] req@ffffa06c07e51880 x1876700024683648/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762591 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762566.519444:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51880 x1876700024683648/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762591 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762566.519452:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762566.519457:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762566.519461:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024683648:192.168.203.107@tcp:38:kworker.0 00010000:00010000:0.0:1789762566.774987:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762567.030942:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762567.798956:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762568.054938:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762568.823024:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762569.078970:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789762569.846968:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762570.102920:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762570.871006:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762571.126945:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789762571.638937:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762571.638945:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789762571.638948:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762571.638956:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2005 00000100:00080000:0.0:1789762571.638964:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762571.638969:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789762571.639000:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0000 x1876700024683776/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762571.639021:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762571.639025:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762571.639026:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762571.639029:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2005 00000100:00080000:0.0:1789762571.639031:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762571.639033:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789762571.639056:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762571.639066:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d3b80 x1876700024683904/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762571.639075:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762571.639078:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762571.639079:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762571.639082:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2005 00000100:00080000:0.0:1789762571.639085:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762571.639087:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789762571.639091:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762571.639096:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024683776:192.168.203.107@tcp:250:kworker.0 00000100:00100000:0.0:1789762571.639101:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d2300 x1876700024684032/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789762571.639151:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762571.639182:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762571/real 1789762571] req@ffffa06c091d0000 x1876700024683776/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762596 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762571.639195:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0000 x1876700024683776/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762596 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762571.639203:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762571.639206:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762571.639215:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024683776:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789762571.639221:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024683904:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789762571.639236:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762571.639245:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762571/real 1789762571] req@ffffa06c091d3b80 x1876700024683904/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762596 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762571.639253:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d3b80 x1876700024683904/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762596 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762571.639258:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762571.639260:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762571.639263:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024683904:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789762571.639280:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024684032:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789762571.639290:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762571.639296:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762571/real 1789762571] req@ffffa06c091d2300 x1876700024684032/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762596 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762571.639303:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d2300 x1876700024684032/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762596 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762571.639308:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762571.639310:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762571.639312:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024684032:192.168.203.107@tcp:38:kworker.0 00010000:00010000:1.0:1789762571.894960:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762572.150915:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762572.919017:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762573.174987:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762573.943010:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762574.198925:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762574.967099:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762575.222958:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789762575.991050:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762576.246928:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789762576.758973:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762576.758982:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789762576.758986:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762576.758994:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2010 00000100:00080000:2.0:1789762576.759004:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762576.759010:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789762576.759043:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51880 x1876700024684160/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762576.759066:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762576.759069:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762576.759070:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762576.759073:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2010 00000100:00080000:2.0:1789762576.759088:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762576.759092:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789762576.759100:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762576.759107:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024684160:192.168.203.107@tcp:250:kworker.0 00000100:00100000:2.0:1789762576.759113:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e52300 x1876700024684288/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762576.759123:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762576.759126:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762576.759128:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762576.759130:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2010 00000100:00080000:2.0:1789762576.759133:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762576.759135:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789762576.759138:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762576.759147:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51180 x1876700024684416/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789762576.759169:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762576.759210:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762576/real 1789762576] req@ffffa06c07e51880 x1876700024684160/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762601 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762576.759225:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51880 x1876700024684160/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762601 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762576.759233:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762576.759236:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762576.759239:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024684160:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789762576.759257:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e52300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024684288:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789762576.759274:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762576.759282:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762576/real 1789762576] req@ffffa06c07e52300 x1876700024684288/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762601 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762576.759290:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e52300 x1876700024684288/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762601 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762576.759296:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762576.759298:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762576.759300:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e52300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024684288:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789762576.759303:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024684416:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789762576.759324:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762576.759331:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762576/real 1789762576] req@ffffa06c07e51180 x1876700024684416/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762601 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762576.759338:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51180 x1876700024684416/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762601 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762576.759344:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762576.759346:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762576.759348:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024684416:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789762577.014977:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762577.270945:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762578.039005:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762578.294931:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789762579.062960:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762579.318990:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762580.087064:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762580.342935:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762581.111008:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762581.366966:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789762581.878980:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762581.878987:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789762581.878990:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762581.878998:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2015 00000100:00080000:2.0:1789762581.879023:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762581.879028:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789762581.879063:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50000 x1876700024684544/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762581.879085:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762581.879088:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762581.879089:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762581.879093:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2015 00000100:00080000:2.0:1789762581.879095:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762581.879097:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789762581.879104:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762581.879112:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51500 x1876700024684672/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762581.879122:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762581.879124:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762581.879126:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762581.879132:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2015 00000100:00080000:2.0:1789762581.879134:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762581.879138:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789762581.879141:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762581.879148:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50e00 x1876700024684800/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789762581.879167:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024684544:192.168.203.107@tcp:250:kworker.0 00000800:00000100:1.0:1789762581.879245:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762581.879288:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762581/real 1789762581] req@ffffa06c07e50000 x1876700024684544/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762606 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762581.879309:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50000 x1876700024684544/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762606 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762581.879322:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762581.879327:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762581.879332:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024684544:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789762581.879342:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024684672:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789762581.879376:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762581.879387:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762581/real 1789762581] req@ffffa06c07e51500 x1876700024684672/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762606 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762581.879413:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51500 x1876700024684672/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762606 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762581.879420:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762581.879423:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762581.879425:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024684672:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789762581.879450:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024684800:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789762581.879466:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762581.879473:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762581/real 1789762581] req@ffffa06c07e50e00 x1876700024684800/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762606 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762581.879482:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50e00 x1876700024684800/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762606 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762581.879487:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762581.879490:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762581.879492:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024684800:192.168.203.107@tcp:38:kworker.0 00010000:00010000:1.0:1789762582.135023:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762582.390926:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762583.158988:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762583.414975:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762584.183035:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762584.439012:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789762585.206998:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762585.462951:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762586.231027:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762586.486947:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789762586.998984:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762586.998994:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789762586.998998:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762586.999011:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2020 00000100:00080000:0.0:1789762586.999024:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762586.999031:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789762586.999078:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d2300 x1876700024684928/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762586.999104:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762586.999107:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762586.999109:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762586.999112:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2020 00000100:00080000:0.0:1789762586.999133:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762586.999136:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789762586.999147:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762586.999157:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d3b80 x1876700024685056/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762586.999168:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762586.999172:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762586.999174:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1789762586.999174:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024684928:192.168.203.107@tcp:250:kworker.0 00000100:00080000:0.0:1789762586.999180:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2020 00000100:00080000:0.0:1789762586.999186:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762586.999189:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789762586.999192:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762586.999200:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0000 x1876700024685184/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789762586.999265:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762586.999310:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762586/real 1789762586] req@ffffa06c091d2300 x1876700024684928/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762611 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762586.999340:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d2300 x1876700024684928/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762611 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762586.999356:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762586.999360:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762586.999363:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024684928:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789762586.999386:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024685056:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789762586.999405:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762586.999414:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762586/real 1789762586] req@ffffa06c091d3b80 x1876700024685056/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762611 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762586.999423:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d3b80 x1876700024685056/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762611 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762586.999429:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762586.999431:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762586.999434:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024685056:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789762586.999437:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024685184:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789762586.999449:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762586.999455:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762586/real 1789762586] req@ffffa06c091d0000 x1876700024685184/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762611 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762586.999462:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0000 x1876700024685184/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762611 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762586.999467:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762586.999470:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762586.999471:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024685184:192.168.203.107@tcp:38:kworker.0 00010000:00010000:1.0:1789762587.254982:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762587.510935:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789762588.279079:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762588.534969:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762589.303005:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762589.558956:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762590.327056:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762590.582938:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789762591.351128:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762591.606983:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789762592.119053:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762592.119077:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789762592.119082:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762592.119095:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2025 00000100:00080000:2.0:1789762592.119109:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762592.119118:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789762592.119168:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e53b80 x1876700024685312/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762592.119204:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762592.119209:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762592.119212:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762592.119217:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2025 00000100:00080000:2.0:1789762592.119221:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762592.119225:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789762592.119237:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762592.119253:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50380 x1876700024685440/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789762592.119296:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e53b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024685312:192.168.203.107@tcp:250:kworker.0 00000100:00080000:2.0:1789762592.119334:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762592.119340:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762592.119344:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762592.119352:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2025 00000100:00080000:2.0:1789762592.119360:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762592.119366:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789762592.119374:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1789762592.119388:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789762592.119397:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51f80 x1876700024685568/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789762592.119451:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762592/real 1789762592] req@ffffa06c07e53b80 x1876700024685312/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762617 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762592.119475:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e53b80 x1876700024685312/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762617 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762592.119509:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762592.119517:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762592.119525:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e53b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024685312:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789762592.119566:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024685440:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789762592.119610:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762592.119629:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762592/real 1789762592] req@ffffa06c07e50380 x1876700024685440/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762617 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762592.119646:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50380 x1876700024685440/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762617 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762592.119657:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762592.119662:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762592.119666:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024685440:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789762592.119677:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024685568:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789762592.119742:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762592.119766:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762592/real 1789762592] req@ffffa06c07e51f80 x1876700024685568/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762617 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762592.119789:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51f80 x1876700024685568/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762617 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762592.119799:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762592.119804:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762592.119808:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024685568:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789762592.375028:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762592.630944:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762593.398946:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762593.654958:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762594.422963:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762594.678965:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762595.446949:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762595.702933:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762596.471008:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762596.727005:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:1.0:1789762597.239034:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762597.239046:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789762597.239051:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762597.239063:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2031 00000100:00080000:1.0:1789762597.239077:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762597.239086:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789762597.239134:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2d500 x1876700024685696/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762597.239180:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762597.239185:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762597.239187:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762597.239193:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2031 00000100:00080000:1.0:1789762597.239198:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762597.239203:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789762597.239218:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762597.239272:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024685696:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789762597.239296:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2ed80 x1876700024685824/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762597.239317:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762597.239322:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762597.239325:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762597.239331:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2031 00000100:00080000:1.0:1789762597.239337:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762597.239344:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789762597.239351:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762597.239363:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2e300 x1876700024685952/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789762597.239394:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789762597.239462:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762597/real 1789762597] req@ffffa06c08e2d500 x1876700024685696/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762622 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762597.239486:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2d500 x1876700024685696/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762622 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762597.239500:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762597.239506:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789762597.239511:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024685696:192.168.203.107@tcp:250:kworker.0 00000100:00100000:2.0:1789762597.239570:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024685824:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789762597.239600:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789762597.239615:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762597/real 1789762597] req@ffffa06c08e2ed80 x1876700024685824/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762622 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762597.239629:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2ed80 x1876700024685824/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762622 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762597.239639:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762597.239644:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789762597.239648:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024685824:192.168.203.107@tcp:38:kworker.0 00000100:00100000:2.0:1789762597.239656:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024685952:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789762597.239677:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789762597.239689:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762597/real 1789762597] req@ffffa06c08e2e300 x1876700024685952/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762622 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762597.239702:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2e300 x1876700024685952/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762622 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762597.239711:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762597.239715:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789762597.239719:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024685952:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789762597.495005:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762597.750958:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762598.518972:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762598.774948:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762599.542993:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762599.798949:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762600.566987:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762600.822930:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762601.591011:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762601.846961:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:1.0:1789762602.358945:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762602.358960:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789762602.358965:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762602.358977:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2036 00000100:00080000:1.0:1789762602.358991:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762602.359001:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789762602.359049:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2c000 x1876700024686080/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762602.359078:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762602.359083:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762602.359085:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762602.359091:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2036 00000100:00080000:1.0:1789762602.359096:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1789762602.359109:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024686080:192.168.203.107@tcp:250:kworker.0 00000100:00080000:1.0:1789762602.359121:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789762602.359134:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762602.359152:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2c700 x1876700024686208/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762602.359166:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762602.359171:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762602.359174:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762602.359177:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2036 00000100:00080000:1.0:1789762602.359182:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1789762602.359187:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00080000:1.0:1789762602.359191:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789762602.359198:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762602.359229:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2ca80 x1876700024686336/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789762602.359255:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762602/real 1789762602] req@ffffa06c08e2c000 x1876700024686080/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762627 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762602.359281:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2c000 x1876700024686080/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762627 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762602.359295:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762602.359301:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789762602.359305:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024686080:192.168.203.107@tcp:250:kworker.0 00000100:00100000:2.0:1789762602.359332:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024686208:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789762602.359355:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789762602.359368:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762602/real 1789762602] req@ffffa06c08e2c700 x1876700024686208/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762627 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762602.359380:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2c700 x1876700024686208/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762627 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762602.359387:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762602.359389:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789762602.359392:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024686208:192.168.203.107@tcp:38:kworker.0 00000100:00100000:2.0:1789762602.359396:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024686336:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789762602.359411:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789762602.359419:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762602/real 1789762602] req@ffffa06c08e2ca80 x1876700024686336/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762627 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762602.359436:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2ca80 x1876700024686336/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762627 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762602.359444:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762602.359447:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789762602.359449:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024686336:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789762602.614963:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762602.870930:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762603.639012:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762603.894956:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789762604.662965:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762604.918955:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762605.686997:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762605.942972:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762606.711008:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762606.966946:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789762607.478964:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762607.478976:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789762607.478979:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762607.478990:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2041 00000100:00080000:2.0:1789762607.479002:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762607.479009:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789762607.479055:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e52680 x1876700024686464/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762607.479082:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762607.479086:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762607.479087:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762607.479090:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2041 00000100:00080000:2.0:1789762607.479117:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762607.479122:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789762607.479131:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e52680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024686464:192.168.203.107@tcp:250:kworker.0 00000008:00100000:2.0:1789762607.479132:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762607.479146:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e53800 x1876700024686592/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762607.479159:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762607.479175:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762607.479177:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762607.479182:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2041 00000100:00080000:2.0:1789762607.479187:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762607.479192:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789762607.479198:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762607.479212:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51c00 x1876700024686720/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789762607.479232:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762607.479284:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762607/real 1789762607] req@ffffa06c07e52680 x1876700024686464/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762632 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762607.479305:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e52680 x1876700024686464/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762632 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762607.479318:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762607.479323:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762607.479328:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e52680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024686464:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789762607.479363:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e53800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024686592:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789762607.479388:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762607.479401:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762607/real 1789762607] req@ffffa06c07e53800 x1876700024686592/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762632 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762607.479414:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e53800 x1876700024686592/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762632 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762607.479423:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762607.479427:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762607.479431:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e53800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024686592:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789762607.479438:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024686720:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789762607.479460:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762607.479472:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762607/real 1789762607] req@ffffa06c07e51c00 x1876700024686720/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762632 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762607.479485:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51c00 x1876700024686720/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762632 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762607.479494:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762607.479497:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762607.479501:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024686720:192.168.203.107@tcp:38:kworker.0 00010000:00010000:2.0:1789762607.735045:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762607.991061:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762608.758963:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762609.014946:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762609.783028:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762610.038926:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789762610.807020:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762611.062916:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762611.830994:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762612.086985:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789762612.598921:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762612.598932:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789762612.598936:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762612.598947:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2046 00000100:00080000:2.0:1789762612.598980:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762612.598987:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789762612.599025:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e53480 x1876700024686848/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762612.599053:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762612.599056:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762612.599057:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762612.599061:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2046 00000100:00080000:2.0:1789762612.599063:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762612.599066:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789762612.599076:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762612.599086:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50a80 x1876700024686976/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762612.599096:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762612.599098:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762612.599100:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762612.599102:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2046 00000100:00080000:2.0:1789762612.599104:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1789762612.599109:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e53480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024686848:192.168.203.107@tcp:250:kworker.0 00000100:00080000:2.0:1789762612.599110:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789762612.599117:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762612.599138:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50700 x1876700024687104/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789762612.599199:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762612.599259:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762612/real 1789762612] req@ffffa06c07e53480 x1876700024686848/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762637 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762612.599278:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e53480 x1876700024686848/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762637 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762612.599287:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762612.599290:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762612.599293:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e53480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024686848:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789762612.599300:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024686976:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789762612.599318:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762612.599329:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762612/real 1789762612] req@ffffa06c07e50a80 x1876700024686976/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762637 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762612.599343:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50a80 x1876700024686976/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762637 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762612.599352:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762612.599356:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762612.599360:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024686976:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789762612.599387:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024687104:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789762612.599412:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762612.599420:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762612/real 1789762612] req@ffffa06c07e50700 x1876700024687104/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762637 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762612.599434:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50700 x1876700024687104/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762637 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762612.599455:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762612.599459:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762612.599463:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024687104:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789762612.854995:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762613.110937:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789762613.878987:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762614.134961:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789762614.903036:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762615.158912:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762615.926961:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762616.182938:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762616.950964:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762617.206908:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789762617.719959:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762617.719967:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789762617.719970:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762617.719979:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2051 00000100:00080000:0.0:1789762617.719987:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762617.719993:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789762617.720023:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d2a00 x1876700024687232/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762617.720045:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762617.720058:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762617.720060:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762617.720064:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2051 00000100:00080000:0.0:1789762617.720068:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762617.720071:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789762617.720078:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762617.720100:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d1500 x1876700024687360/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789762617.720108:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024687232:192.168.203.107@tcp:250:kworker.0 00000100:00080000:0.0:1789762617.720113:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762617.720117:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762617.720118:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762617.720122:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2051 00000100:00080000:0.0:1789762617.720124:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762617.720127:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789762617.720130:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762617.720139:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0e00 x1876700024687488/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789762617.720207:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024687360:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789762617.720229:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024687488:192.168.203.107@tcp:38:kworker.0 00010000:00010000:2.0:1789762617.974993:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762618.230913:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762618.998972:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762619.254923:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789762620.022969:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762620.278912:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000400:00000100:3.0:1789762620.791119:0:2329:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.107:988 00000400:00000100:3.0:1789762620.791158:0:2329:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.107@tcp at host 192.168.203.107 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1789762620.791173:0:2329:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.107@tcp 00000800:00000100:3.0:1789762620.791234:0:2329:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.107@tcp 00000800:00000100:3.0:1789762620.791244:0:2329:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.107@tcp 00000100:00100000:1.0:1789762620.791306:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762617/real 1789762620] req@ffffa06c091d1500 x1876700024687360/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762642 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762620.791333:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d1500 x1876700024687360/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762642 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762620.791347:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762620.791352:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762620.791356:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024687360:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789762620.791364:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762617/real 1789762620] req@ffffa06c091d0e00 x1876700024687488/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762642 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762620.791377:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0e00 x1876700024687488/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762642 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762620.791405:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762620.791409:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762620.791413:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024687488:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789762620.791420:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762617/real 1789762620] req@ffffa06c091d2a00 x1876700024687232/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762642 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762620.791433:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d2a00 x1876700024687232/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762642 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762620.791440:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762620.791450:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762620.791453:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024687232:192.168.203.107@tcp:250:kworker.0 00010000:00010000:2.0:1789762621.047025:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762621.302942:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762622.071106:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762622.326966:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789762622.838934:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762622.838955:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789762622.838958:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762622.838967:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2056 00000100:00080000:2.0:1789762622.838976:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762622.838982:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789762622.839028:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e52d80 x1876700024687616/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762622.839064:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762622.839069:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762622.839071:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762622.839076:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2056 00000100:00080000:2.0:1789762622.839080:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762622.839084:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789762622.839094:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762622.839109:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e53100 x1876700024687744/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789762622.839118:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e52d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024687616:192.168.203.107@tcp:250:kworker.0 00000100:00080000:2.0:1789762622.839139:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762622.839143:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762622.839145:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762622.839150:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2056 00000100:00080000:2.0:1789762622.839155:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762622.839160:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789762622.839166:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762622.839182:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e52a00 x1876700024687872/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789762622.839187:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762622.839241:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762622/real 1789762622] req@ffffa06c07e52d80 x1876700024687616/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762647 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762622.839261:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e52d80 x1876700024687616/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762647 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762622.839273:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762622.839277:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762622.839281:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e52d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024687616:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789762622.839305:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e53100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024687744:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789762622.839341:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762622.839355:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762622/real 1789762622] req@ffffa06c07e53100 x1876700024687744/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762647 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762622.839368:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e53100 x1876700024687744/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762647 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762622.839377:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762622.839380:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762622.839384:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e53100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024687744:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789762622.839391:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e52a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024687872:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789762622.839412:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762622.839425:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762622/real 1789762622] req@ffffa06c07e52a00 x1876700024687872/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762647 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762622.839438:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e52a00 x1876700024687872/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762647 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762622.839446:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762622.839449:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762622.839452:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e52a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024687872:192.168.203.107@tcp:38:kworker.0 00010000:00010000:0.0:1789762623.095071:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762623.350917:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789762624.118968:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762624.374921:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762625.142961:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762625.398965:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789762626.167017:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762626.422911:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789762627.191067:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762627.446969:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789762627.958933:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762627.958944:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789762627.958949:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762627.958961:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2061 00000100:00080000:2.0:1789762627.958974:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762627.958982:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789762627.959045:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e52a00 x1876700024688000/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762627.959072:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762627.959075:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762627.959076:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762627.959099:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2061 00000100:00080000:2.0:1789762627.959102:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762627.959106:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789762627.959116:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762627.959128:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e53100 x1876700024688128/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762627.959141:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762627.959144:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762627.959146:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762627.959149:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2061 00000100:00080000:2.0:1789762627.959151:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762627.959154:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789762627.959156:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762627.959165:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e52d80 x1876700024688256/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789762627.959187:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e52a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024688000:192.168.203.107@tcp:250:kworker.0 00000800:00000100:1.0:1789762627.959288:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762627.959338:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762627/real 1789762627] req@ffffa06c07e52a00 x1876700024688000/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762652 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762627.959358:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e52a00 x1876700024688000/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762652 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762627.959371:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762627.959375:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762627.959380:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e52a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024688000:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789762627.959388:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e53100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024688128:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789762627.959408:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762627.959420:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762627/real 1789762627] req@ffffa06c07e53100 x1876700024688128/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762652 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762627.959432:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e53100 x1876700024688128/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762652 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762627.959440:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762627.959444:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762627.959447:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e53100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024688128:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789762627.959472:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e52d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024688256:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789762627.959488:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762627.959498:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762627/real 1789762627] req@ffffa06c07e52d80 x1876700024688256/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762652 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762627.959526:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e52d80 x1876700024688256/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762652 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762627.959535:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762627.959538:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762627.959541:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e52d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024688256:192.168.203.107@tcp:38:kworker.0 00010000:00010000:2.0:1789762628.215081:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762628.470944:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762629.239020:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762629.494949:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789762630.262957:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762630.518904:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762631.287008:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762631.542923:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762632.310959:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762632.566930:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:1.0:1789762633.078940:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762633.078948:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789762633.078952:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762633.078960:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2066 00000100:00080000:1.0:1789762633.078983:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762633.078991:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789762633.079026:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2d180 x1876700024688384/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762633.079050:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762633.079054:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762633.079055:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762633.079059:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2066 00000100:00080000:1.0:1789762633.079062:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762633.079080:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789762633.079088:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024688384:192.168.203.107@tcp:250:kworker.0 00000008:00100000:1.0:1789762633.079089:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762633.079106:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2ca80 x1876700024688512/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762633.079117:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762633.079120:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762633.079121:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762633.079124:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2066 00000100:00080000:1.0:1789762633.079129:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762633.079132:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789762633.079135:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762633.079143:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2c700 x1876700024688640/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789762633.079151:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789762633.079190:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762633/real 1789762633] req@ffffa06c08e2d180 x1876700024688384/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762658 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762633.079205:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2d180 x1876700024688384/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762658 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762633.079225:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762633.079229:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789762633.079232:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024688384:192.168.203.107@tcp:250:kworker.0 00000100:00100000:2.0:1789762633.079252:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024688512:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789762633.079271:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789762633.079280:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762633/real 1789762633] req@ffffa06c08e2ca80 x1876700024688512/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762658 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762633.079289:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2ca80 x1876700024688512/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762658 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762633.079296:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762633.079298:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789762633.079301:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024688512:192.168.203.107@tcp:38:kworker.0 00000100:00100000:2.0:1789762633.079305:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024688640:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789762633.079316:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789762633.079322:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762633/real 1789762633] req@ffffa06c08e2c700 x1876700024688640/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762658 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762633.079330:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2c700 x1876700024688640/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762658 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762633.079336:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762633.079338:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789762633.079341:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024688640:192.168.203.107@tcp:38:kworker.0 00010000:00010000:0.0:1789762633.334991:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762633.590946:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762634.359040:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762634.614926:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762635.383009:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762635.639035:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762636.407013:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762636.662925:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762637.430984:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762637.686921:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:1.0:1789762638.198916:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762638.198924:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789762638.198927:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762638.198948:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2072 00000100:00080000:1.0:1789762638.198957:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762638.198963:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789762638.198995:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2c000 x1876700024688768/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762638.199032:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762638.199035:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762638.199037:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762638.199040:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2072 00000100:00080000:1.0:1789762638.199043:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762638.199046:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789762638.199053:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762638.199063:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2e300 x1876700024688896/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762638.199072:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1789762638.199076:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024688768:192.168.203.107@tcp:250:kworker.0 00000100:00080000:1.0:1789762638.199077:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762638.199080:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762638.199088:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2072 00000100:00080000:1.0:1789762638.199090:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762638.199093:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789762638.199095:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762638.199104:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2ed80 x1876700024689024/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789762638.199138:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789762638.199173:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762638/real 1789762638] req@ffffa06c08e2c000 x1876700024688768/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762663 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762638.199188:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2c000 x1876700024688768/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762663 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762638.199197:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762638.199200:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789762638.199204:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024688768:192.168.203.107@tcp:250:kworker.0 00000100:00100000:2.0:1789762638.199223:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024688896:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789762638.199252:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789762638.199269:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762638/real 1789762638] req@ffffa06c08e2e300 x1876700024688896/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762663 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762638.199283:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2e300 x1876700024688896/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762663 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762638.199294:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762638.199299:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789762638.199303:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024688896:192.168.203.107@tcp:38:kworker.0 00000100:00100000:2.0:1789762638.199311:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024689024:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789762638.199348:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789762638.199365:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762638/real 1789762638] req@ffffa06c08e2ed80 x1876700024689024/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762663 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762638.199382:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2ed80 x1876700024689024/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762663 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762638.199392:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762638.199397:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789762638.199401:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024689024:192.168.203.107@tcp:38:kworker.0 00010000:00010000:2.0:1789762638.455098:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762638.711025:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762639.479056:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762639.735016:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762640.502966:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762640.758933:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762641.526977:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762641.782976:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762642.551043:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762642.806982:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:1.0:1789762643.318985:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762643.318997:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789762643.319002:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762643.319013:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2077 00000100:00080000:1.0:1789762643.319024:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762643.319032:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789762643.319071:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2d500 x1876700024689152/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762643.319103:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762643.319107:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762643.319109:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762643.319114:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2077 00000100:00080000:1.0:1789762643.319118:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762643.319123:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789762643.319134:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762643.319155:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2ed80 x1876700024689280/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789762643.319157:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024689152:192.168.203.107@tcp:250:kworker.0 00000100:00080000:1.0:1789762643.319178:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762643.319183:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762643.319185:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762643.319191:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2077 00000100:00080000:1.0:1789762643.319196:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762643.319201:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789762643.319208:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1789762643.319221:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762643.319243:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2e300 x1876700024689408/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789762643.319277:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762643/real 1789762643] req@ffffa06c08e2d500 x1876700024689152/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762668 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762643.319306:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2d500 x1876700024689152/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762668 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762643.319316:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762643.319320:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789762643.319323:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024689152:192.168.203.107@tcp:250:kworker.0 00000100:00100000:2.0:1789762643.319345:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024689280:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789762643.319374:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789762643.319404:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762643/real 1789762643] req@ffffa06c08e2ed80 x1876700024689280/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762668 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762643.319422:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2ed80 x1876700024689280/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762668 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762643.319431:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762643.319434:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789762643.319437:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024689280:192.168.203.107@tcp:38:kworker.0 00000100:00100000:2.0:1789762643.319443:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024689408:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789762643.319463:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789762643.319472:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762643/real 1789762643] req@ffffa06c08e2e300 x1876700024689408/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762668 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762643.319493:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2e300 x1876700024689408/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762668 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762643.319500:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762643.319503:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789762643.319505:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024689408:192.168.203.107@tcp:38:kworker.0 00010000:00010000:1.0:1789762643.575028:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762643.830983:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762644.599136:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762644.855008:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762645.623091:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762645.878969:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762646.647051:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762646.903012:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762647.671085:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762647.926953:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789762648.438998:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762648.439014:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789762648.439018:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762648.439030:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2082 00000100:00080000:2.0:1789762648.439042:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762648.439073:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789762648.439113:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50700 x1876700024689536/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762648.439144:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762648.439147:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762648.439149:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762648.439152:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2082 00000100:00080000:2.0:1789762648.439157:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762648.439160:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789762648.439169:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762648.439181:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50a80 x1876700024689664/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789762648.439193:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024689536:192.168.203.107@tcp:250:kworker.0 00000100:00080000:2.0:1789762648.439194:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762648.439202:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762648.439207:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762648.439213:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2082 00000100:00080000:2.0:1789762648.439216:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762648.439218:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789762648.439222:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762648.439232:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e53480 x1876700024689792/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789762648.439271:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762648.439310:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762648/real 1789762648] req@ffffa06c07e50700 x1876700024689536/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762673 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762648.439325:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50700 x1876700024689536/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762673 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762648.439333:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762648.439336:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762648.439339:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024689536:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789762648.439358:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024689664:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789762648.439374:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762648.439382:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762648/real 1789762648] req@ffffa06c07e50a80 x1876700024689664/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762673 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762648.439390:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50a80 x1876700024689664/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762673 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762648.439396:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762648.439411:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762648.439414:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024689664:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789762648.439419:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e53480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024689792:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789762648.439433:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762648.439440:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762648/real 1789762648] req@ffffa06c07e53480 x1876700024689792/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762673 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762648.439448:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e53480 x1876700024689792/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762673 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762648.439453:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762648.439455:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762648.439456:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e53480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024689792:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789762648.695020:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762648.950941:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789762649.719026:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762649.974920:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762650.743003:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762650.998941:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762651.766965:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762652.022909:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789762652.791044:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762653.046931:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789762653.558931:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762653.558966:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789762653.558971:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762653.558984:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2087 00000100:00080000:2.0:1789762653.558998:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762653.559007:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789762653.559052:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51c00 x1876700024689920/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762653.559084:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762653.559088:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762653.559090:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762653.559095:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2087 00000100:00080000:2.0:1789762653.559099:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762653.559103:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789762653.559115:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762653.559130:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e53800 x1876700024690048/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762653.559143:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762653.559148:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762653.559150:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762653.559156:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2087 00000100:00100000:1.0:1789762653.559164:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024689920:192.168.203.107@tcp:250:kworker.0 00000100:00080000:2.0:1789762653.559165:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762653.559178:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789762653.559185:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762653.559201:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e52680 x1876700024690176/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789762653.559245:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762653.559283:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762653/real 1789762653] req@ffffa06c07e51c00 x1876700024689920/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762678 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762653.559297:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51c00 x1876700024689920/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762678 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762653.559321:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762653.559325:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762653.559328:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024689920:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789762653.559333:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e53800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024690048:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789762653.559351:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762653.559359:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762653/real 1789762653] req@ffffa06c07e53800 x1876700024690048/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762678 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762653.559367:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e53800 x1876700024690048/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762678 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762653.559373:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762653.559375:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762653.559377:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e53800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024690048:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789762653.559396:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e52680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024690176:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789762653.559407:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762653.559413:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762653/real 1789762653] req@ffffa06c07e52680 x1876700024690176/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762678 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762653.559420:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e52680 x1876700024690176/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762678 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762653.559425:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762653.559427:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762653.559429:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e52680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024690176:192.168.203.107@tcp:38:kworker.0 00010000:00010000:0.0:1789762653.815036:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762654.070947:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762654.839036:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762655.094927:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789762655.863040:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762656.118913:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789762656.886974:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762657.142926:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762657.910979:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762658.166980:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789762658.678970:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762658.678981:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789762658.678985:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762658.678995:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2092 00000100:00080000:2.0:1789762658.679007:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762658.679015:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789762658.679056:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51f80 x1876700024690304/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762658.679086:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762658.679090:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762658.679092:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762658.679096:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2092 00000100:00080000:2.0:1789762658.679100:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762658.679103:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789762658.679113:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762658.679125:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50380 x1876700024690432/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762658.679136:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762658.679140:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762658.679142:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762658.679146:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2092 00000100:00080000:2.0:1789762658.679150:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762658.679154:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789762658.679180:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024690304:192.168.203.107@tcp:250:kworker.0 00000008:00100000:2.0:1789762658.679180:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762658.679200:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e53b80 x1876700024690560/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789762658.679259:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762658.679303:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762658/real 1789762658] req@ffffa06c07e51f80 x1876700024690304/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762683 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762658.679322:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51f80 x1876700024690304/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762683 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762658.679347:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762658.679352:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762658.679357:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024690304:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789762658.679365:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024690432:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789762658.679386:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762658.679399:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762658/real 1789762658] req@ffffa06c07e50380 x1876700024690432/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762683 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762658.679412:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50380 x1876700024690432/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762683 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762658.679420:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762658.679423:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762658.679437:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024690432:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789762658.679463:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e53b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024690560:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789762658.679485:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762658.679496:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762658/real 1789762658] req@ffffa06c07e53b80 x1876700024690560/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762683 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762658.679508:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e53b80 x1876700024690560/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762683 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762658.679515:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762658.679518:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762658.679521:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e53b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024690560:192.168.203.107@tcp:38:kworker.0 00010000:00010000:0.0:1789762658.935126:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762659.190930:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762659.958955:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762660.214929:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762660.982948:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762661.238915:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789762662.006959:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762662.262953:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762663.030984:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762663.287898:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789762663.798912:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762663.798920:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789762663.798923:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762663.798932:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2097 00000100:00080000:2.0:1789762663.798941:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762663.798947:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789762663.798989:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50e00 x1876700024690688/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762663.799008:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762663.799011:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762663.799013:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762663.799016:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2097 00000100:00080000:2.0:1789762663.799018:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762663.799020:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789762663.799028:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762663.799036:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51500 x1876700024690816/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762663.799045:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762663.799048:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762663.799056:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762663.799061:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2097 00000100:00080000:2.0:1789762663.799064:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762663.799069:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789762663.799070:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024690688:192.168.203.107@tcp:250:kworker.0 00000008:00100000:2.0:1789762663.799073:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762663.799083:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50000 x1876700024690944/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789762663.799126:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762663.799157:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762663/real 1789762663] req@ffffa06c07e50e00 x1876700024690688/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762688 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762663.799170:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50e00 x1876700024690688/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762688 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762663.799178:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762663.799181:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762663.799184:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024690688:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789762663.799196:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024690816:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789762663.799212:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762663.799228:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762663/real 1789762663] req@ffffa06c07e51500 x1876700024690816/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762688 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762663.799236:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51500 x1876700024690816/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762688 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762663.799243:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762663.799246:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762663.799250:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024690816:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789762663.799273:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024690944:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789762663.799293:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762663.799305:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762663/real 1789762663] req@ffffa06c07e50000 x1876700024690944/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762688 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762663.799318:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50000 x1876700024690944/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762688 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762663.799326:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762663.799330:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762663.799333:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024690944:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789762664.054937:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762664.310901:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762665.078989:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762665.334935:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762666.103014:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762666.358961:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762667.126959:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762667.382904:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762668.150979:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762668.406939:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789762668.919019:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762668.919032:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789762668.919038:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762668.919052:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2102 00000100:00080000:0.0:1789762668.919066:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762668.919075:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789762668.919123:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d2680 x1876700024691072/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762668.919165:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762668.919172:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762668.919176:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762668.919185:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2102 00000100:00080000:0.0:1789762668.919192:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762668.919200:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789762668.919215:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762668.919242:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0380 x1876700024691200/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762668.919277:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762668.919282:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762668.919285:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762668.919291:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2102 00000100:00080000:0.0:1789762668.919296:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762668.919303:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789762668.919311:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762668.919314:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024691072:192.168.203.107@tcp:250:kworker.0 00000100:00100000:0.0:1789762668.919350:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d3100 x1876700024691328/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789762668.919396:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762668.919443:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762668/real 1789762668] req@ffffa06c091d2680 x1876700024691072/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762693 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762668.919498:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d2680 x1876700024691072/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762693 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762668.919512:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762668.919517:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762668.919522:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024691072:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789762668.919532:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024691200:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789762668.919564:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762668.919580:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762668/real 1789762668] req@ffffa06c091d0380 x1876700024691200/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762693 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762668.919595:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0380 x1876700024691200/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762693 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762668.919605:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762668.919609:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762668.919613:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024691200:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789762668.919656:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024691328:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789762668.919673:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762668.919681:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762668/real 1789762668] req@ffffa06c091d3100 x1876700024691328/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762693 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762668.919691:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d3100 x1876700024691328/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762693 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762668.919733:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762668.919736:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762668.919740:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024691328:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789762669.175225:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762669.430905:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762670.199001:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762670.454934:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762671.222967:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762671.478994:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762672.247003:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762672.502965:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762673.270985:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762673.526922:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789762674.038934:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762674.038942:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789762674.038946:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762674.038955:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2107 00000100:00080000:2.0:1789762674.038963:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762674.038970:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789762674.039002:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51180 x1876700024691456/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762674.039034:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762674.039037:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762674.039039:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762674.039056:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2107 00000100:00080000:2.0:1789762674.039059:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762674.039066:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789762674.039067:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024691456:192.168.203.107@tcp:250:kworker.0 00000008:00100000:2.0:1789762674.039074:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762674.039089:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e52300 x1876700024691584/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762674.039099:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762674.039102:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762674.039104:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762674.039107:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2107 00000100:00080000:2.0:1789762674.039110:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762674.039114:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789762674.039118:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762674.039128:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51880 x1876700024691712/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789762674.039136:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789762674.039184:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762674/real 1789762674] req@ffffa06c07e51180 x1876700024691456/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762699 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762674.039198:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51180 x1876700024691456/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762699 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762674.039206:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762674.039209:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789762674.039211:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024691456:192.168.203.107@tcp:250:kworker.0 00000100:00100000:0.0:1789762674.039234:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e52300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024691584:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789762674.039257:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789762674.039271:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762674/real 1789762674] req@ffffa06c07e52300 x1876700024691584/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762699 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762674.039283:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e52300 x1876700024691584/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762699 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762674.039292:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762674.039295:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789762674.039300:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e52300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024691584:192.168.203.107@tcp:38:kworker.0 00000100:00100000:0.0:1789762674.039306:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024691712:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789762674.039336:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789762674.039348:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762674/real 1789762674] req@ffffa06c07e51880 x1876700024691712/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762699 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762674.039360:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51880 x1876700024691712/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762699 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762674.039369:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762674.039373:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789762674.039376:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024691712:192.168.203.107@tcp:38:kworker.0 00010000:00010000:0.0:1789762674.294973:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762674.550945:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762675.318989:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762675.574920:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762676.343058:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762676.598938:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789762677.367035:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762677.623016:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762678.390979:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762678.646960:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789762679.158971:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762679.158978:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789762679.158982:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762679.158990:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2112 00000100:00080000:0.0:1789762679.158999:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762679.159005:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789762679.159037:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0a80 x1876700024691840/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762679.159065:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762679.159067:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762679.159069:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762679.159071:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2112 00000100:00080000:0.0:1789762679.159074:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762679.159090:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789762679.159097:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762679.159106:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0700 x1876700024691968/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762679.159114:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762679.159117:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762679.159118:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762679.159122:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2112 00000100:00080000:0.0:1789762679.159125:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762679.159128:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789762679.159130:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762679.159144:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d1f80 x1876700024692096/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789762679.159165:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024691840:192.168.203.107@tcp:250:kworker.0 00000800:00000100:1.0:1789762679.159248:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762679.159291:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762679/real 1789762679] req@ffffa06c091d0a80 x1876700024691840/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762704 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762679.159307:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0a80 x1876700024691840/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762704 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762679.159317:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762679.159321:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762679.159325:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024691840:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789762679.159330:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024691968:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789762679.159344:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762679.159353:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762679/real 1789762679] req@ffffa06c091d0700 x1876700024691968/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762704 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762679.159373:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0700 x1876700024691968/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762704 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762679.159379:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762679.159382:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762679.159384:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024691968:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789762679.159408:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024692096:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789762679.159422:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762679.159428:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762679/real 1789762679] req@ffffa06c091d1f80 x1876700024692096/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762704 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762679.159436:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d1f80 x1876700024692096/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762704 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762679.159441:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762679.159443:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762679.159445:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024692096:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789762679.414991:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762679.670954:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762680.438975:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762680.694944:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762681.462994:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762681.718965:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762682.486984:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762682.742916:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762683.510983:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762683.766940:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789762684.278915:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762684.278923:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789762684.278926:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762684.278934:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2118 00000100:00080000:0.0:1789762684.278942:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762684.278949:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789762684.278979:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d1f80 x1876700024692224/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762684.278999:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762684.279001:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762684.279012:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762684.279015:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2118 00000100:00080000:0.0:1789762684.279018:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762684.279021:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789762684.279029:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762684.279039:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0700 x1876700024692352/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789762684.279048:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024692224:192.168.203.107@tcp:250:kworker.0 00000100:00080000:0.0:1789762684.279050:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762684.279056:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762684.279058:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762684.279061:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2118 00000100:00080000:0.0:1789762684.279063:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762684.279065:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789762684.279069:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762684.279077:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0a80 x1876700024692480/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789762684.279145:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024692352:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789762684.279187:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024692480:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789762684.534961:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762684.790933:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762685.558979:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762685.814954:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762686.582957:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762686.838917:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000400:00000100:2.0:1789762687.351092:0:2330:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.107:988 00000400:00000100:2.0:1789762687.351146:0:2330:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.107@tcp at host 192.168.203.107 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1789762687.351165:0:2330:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.107@tcp 00000800:00000100:2.0:1789762687.351203:0:2330:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.107@tcp 00000800:00000100:2.0:1789762687.351212:0:2330:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.107@tcp 00000100:00100000:1.0:1789762687.351252:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762684/real 1789762687] req@ffffa06c091d0700 x1876700024692352/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762709 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762687.351276:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0700 x1876700024692352/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762709 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762687.351288:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762687.351294:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762687.351300:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024692352:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789762687.351309:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762684/real 1789762687] req@ffffa06c091d0a80 x1876700024692480/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762709 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762687.351323:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0a80 x1876700024692480/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762709 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762687.351331:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762687.351335:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762687.351339:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024692480:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789762687.351347:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762684/real 1789762687] req@ffffa06c091d1f80 x1876700024692224/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762709 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762687.351366:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d1f80 x1876700024692224/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762709 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762687.351393:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762687.351397:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762687.351401:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024692224:192.168.203.107@tcp:250:kworker.0 00010000:00010000:0.0:1789762687.607026:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762687.862976:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789762688.630957:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762688.886904:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:1.0:1789762689.399063:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762689.399074:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789762689.399078:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762689.399089:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2123 00000100:00080000:1.0:1789762689.399099:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762689.399107:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789762689.399136:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2c000 x1876700024692608/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762689.399167:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762689.399172:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762689.399174:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762689.399178:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2123 00000100:00080000:1.0:1789762689.399183:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762689.399187:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789762689.399197:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762689.399215:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024692608:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789762689.399216:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2c700 x1876700024692736/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762689.399235:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762689.399249:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762689.399252:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762689.399257:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2123 00000100:00080000:1.0:1789762689.399280:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762689.399285:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000800:00000100:0.0:1789762689.399292:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000008:00100000:1.0:1789762689.399295:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762689.399312:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2ca80 x1876700024692864/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762689.399340:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762689/real 1789762689] req@ffffa06c08e2c000 x1876700024692608/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762714 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762689.399359:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2c000 x1876700024692608/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762714 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762689.399379:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762689.399384:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789762689.399388:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024692608:192.168.203.107@tcp:250:kworker.0 00000100:00100000:0.0:1789762689.399412:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024692736:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789762689.399438:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789762689.399452:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762689/real 1789762689] req@ffffa06c08e2c700 x1876700024692736/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762714 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762689.399465:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2c700 x1876700024692736/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762714 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762689.399473:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762689.399476:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789762689.399479:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024692736:192.168.203.107@tcp:38:kworker.0 00000100:00100000:0.0:1789762689.399485:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024692864:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789762689.399507:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789762689.399519:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762689/real 1789762689] req@ffffa06c08e2ca80 x1876700024692864/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762714 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762689.399543:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2ca80 x1876700024692864/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762714 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762689.399566:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762689.399570:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789762689.399574:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024692864:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789762689.654956:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762689.910962:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789762690.678971:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762690.934921:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789762691.702997:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762691.958945:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762692.726961:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762692.982970:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762693.750977:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762694.006922:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:1.0:1789762694.518914:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762694.518921:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789762694.518924:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762694.518932:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2128 00000100:00080000:1.0:1789762694.518941:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762694.518947:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789762694.518981:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2d180 x1876700024692992/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762694.519001:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762694.519004:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762694.519005:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762694.519008:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2128 00000100:00080000:1.0:1789762694.519023:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762694.519026:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789762694.519033:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762694.519043:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2ca80 x1876700024693120/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762694.519051:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762694.519054:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762694.519055:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762694.519058:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2128 00000100:00080000:1.0:1789762694.519060:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762694.519063:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789762694.519066:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762694.519072:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2c700 x1876700024693248/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762694.519078:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024692992:192.168.203.107@tcp:250:kworker.0 00000800:00000100:0.0:1789762694.519133:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789762694.519165:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762694/real 1789762694] req@ffffa06c08e2d180 x1876700024692992/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762719 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762694.519188:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2d180 x1876700024692992/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762719 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762694.519196:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762694.519199:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789762694.519201:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024692992:192.168.203.107@tcp:250:kworker.0 00000100:00100000:0.0:1789762694.519207:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024693120:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789762694.519220:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789762694.519228:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762694/real 1789762694] req@ffffa06c08e2ca80 x1876700024693120/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762719 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762694.519236:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2ca80 x1876700024693120/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762719 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762694.519241:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762694.519243:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789762694.519245:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024693120:192.168.203.107@tcp:38:kworker.0 00000100:00100000:0.0:1789762694.519262:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024693248:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789762694.519284:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789762694.519291:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762694/real 1789762694] req@ffffa06c08e2c700 x1876700024693248/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762719 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762694.519299:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2c700 x1876700024693248/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762719 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762694.519304:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762694.519306:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789762694.519308:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024693248:192.168.203.107@tcp:38:kworker.0 00010000:00010000:2.0:1789762694.774973:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762695.030926:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762695.799053:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762696.054978:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762696.823029:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762697.078931:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762697.847016:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762698.102945:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762698.871029:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762699.126918:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789762699.639046:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762699.639095:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789762699.639106:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762699.639118:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2133 00000100:00080000:2.0:1789762699.639138:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762699.639151:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789762699.639224:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51880 x1876700024693376/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762699.639258:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762699.639261:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762699.639270:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762699.639274:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2133 00000100:00080000:2.0:1789762699.639277:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762699.639281:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789762699.639299:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762699.639326:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e52300 x1876700024693504/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762699.639340:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024693376:192.168.203.107@tcp:250:kworker.0 00000100:00080000:2.0:1789762699.639343:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762699.639358:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762699.639359:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762699.639364:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2133 00000100:00080000:2.0:1789762699.639376:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762699.639381:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789762699.639386:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1789762699.639411:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789762699.639419:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51180 x1876700024693632/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762699.639462:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762699/real 1789762699] req@ffffa06c07e51880 x1876700024693376/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762724 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762699.639496:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51880 x1876700024693376/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762724 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762699.639509:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762699.639513:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789762699.639516:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024693376:192.168.203.107@tcp:250:kworker.0 00000100:00100000:0.0:1789762699.639537:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e52300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024693504:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789762699.639555:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789762699.639564:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762699/real 1789762699] req@ffffa06c07e52300 x1876700024693504/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762724 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762699.639590:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e52300 x1876700024693504/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762724 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762699.639596:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762699.639599:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789762699.639601:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e52300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024693504:192.168.203.107@tcp:38:kworker.0 00000100:00100000:0.0:1789762699.639605:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024693632:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789762699.639618:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789762699.639625:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762699/real 1789762699] req@ffffa06c07e51180 x1876700024693632/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762724 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762699.639632:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51180 x1876700024693632/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762724 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762699.639637:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762699.639639:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789762699.639640:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024693632:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789762699.894968:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762700.150951:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789762700.919020:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762701.174914:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762701.942946:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762702.198962:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762702.967025:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762703.222947:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762703.990974:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762704.246951:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789762704.758980:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762704.758992:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789762704.758997:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762704.759008:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2138 00000100:00080000:2.0:1789762704.759020:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762704.759028:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789762704.759075:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50000 x1876700024693760/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762704.759098:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762704.759101:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762704.759102:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762704.759106:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2138 00000100:00080000:2.0:1789762704.759108:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762704.759111:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789762704.759121:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762704.759132:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51500 x1876700024693888/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762704.759147:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024693760:192.168.203.107@tcp:250:kworker.0 00000100:00080000:2.0:1789762704.759148:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762704.759154:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762704.759161:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762704.759169:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2138 00000100:00080000:2.0:1789762704.759173:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762704.759198:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789762704.759205:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762704.759221:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50e00 x1876700024694016/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789762704.759235:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789762704.759279:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762704/real 1789762704] req@ffffa06c07e50000 x1876700024693760/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762729 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762704.759299:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50000 x1876700024693760/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762729 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762704.759321:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762704.759325:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789762704.759330:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024693760:192.168.203.107@tcp:250:kworker.0 00000100:00100000:0.0:1789762704.759356:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024693888:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789762704.759379:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789762704.759391:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762704/real 1789762704] req@ffffa06c07e51500 x1876700024693888/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762729 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762704.759404:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51500 x1876700024693888/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762729 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762704.759413:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762704.759417:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789762704.759420:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024693888:192.168.203.107@tcp:38:kworker.0 00000100:00100000:0.0:1789762704.759426:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024694016:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789762704.759459:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789762704.759470:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762704/real 1789762704] req@ffffa06c07e50e00 x1876700024694016/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762729 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762704.759482:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50e00 x1876700024694016/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762729 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762704.759491:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762704.759494:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789762704.759498:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024694016:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789762705.015038:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762705.270970:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762706.038944:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762706.294946:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762707.062992:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762707.318963:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789762708.086993:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762708.342932:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762709.110956:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762709.366977:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789762709.878925:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762709.878933:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789762709.878936:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762709.878944:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2143 00000100:00080000:2.0:1789762709.878958:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762709.878965:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789762709.878995:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e53b80 x1876700024694144/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762709.879016:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762709.879018:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762709.879020:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762709.879023:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2143 00000100:00080000:2.0:1789762709.879025:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762709.879028:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789762709.879035:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762709.879044:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50380 x1876700024694272/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762709.879054:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e53b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024694144:192.168.203.107@tcp:250:kworker.0 00000100:00080000:2.0:1789762709.879054:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762709.879062:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762709.879077:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762709.879082:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2143 00000100:00080000:2.0:1789762709.879085:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762709.879089:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789762709.879093:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762709.879101:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51f80 x1876700024694400/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789762709.879113:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789762709.879147:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762709/real 1789762709] req@ffffa06c07e53b80 x1876700024694144/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762734 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762709.879163:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e53b80 x1876700024694144/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762734 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762709.879171:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762709.879175:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789762709.879180:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e53b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024694144:192.168.203.107@tcp:250:kworker.0 00000100:00100000:0.0:1789762709.879198:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024694272:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789762709.879216:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789762709.879226:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762709/real 1789762709] req@ffffa06c07e50380 x1876700024694272/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762734 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762709.879234:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50380 x1876700024694272/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762734 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762709.879241:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762709.879244:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789762709.879248:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024694272:192.168.203.107@tcp:38:kworker.0 00000100:00100000:0.0:1789762709.879253:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024694400:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789762709.879269:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789762709.879288:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762709/real 1789762709] req@ffffa06c07e51f80 x1876700024694400/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762734 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762709.879296:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51f80 x1876700024694400/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762734 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762709.879303:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762709.879306:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789762709.879309:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024694400:192.168.203.107@tcp:38:kworker.0 00010000:00010000:2.0:1789762710.134990:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762710.390930:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762711.158985:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762711.414961:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762712.182952:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762712.438908:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762713.207022:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762713.462952:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789762714.230973:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762714.486928:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789762714.998938:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762714.998951:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789762714.998956:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762714.998970:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2148 00000100:00080000:2.0:1789762714.998983:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762714.998993:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789762714.999037:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e52680 x1876700024694528/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762714.999066:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762714.999069:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762714.999070:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762714.999074:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2148 00000100:00080000:2.0:1789762714.999077:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762714.999081:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789762714.999092:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762714.999101:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e52680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024694528:192.168.203.107@tcp:250:kworker.0 00000100:00100000:2.0:1789762714.999124:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e53800 x1876700024694656/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762714.999136:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762714.999138:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762714.999140:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762714.999143:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2148 00000100:00080000:2.0:1789762714.999147:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762714.999150:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789762714.999154:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762714.999161:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51c00 x1876700024694784/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789762714.999176:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789762714.999218:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762714/real 1789762714] req@ffffa06c07e52680 x1876700024694528/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762739 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762714.999233:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e52680 x1876700024694528/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762739 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762714.999242:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762714.999245:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789762714.999249:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e52680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024694528:192.168.203.107@tcp:250:kworker.0 00000100:00100000:0.0:1789762714.999272:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e53800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024694656:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789762714.999295:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789762714.999309:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762714/real 1789762714] req@ffffa06c07e53800 x1876700024694656/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762739 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762714.999324:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e53800 x1876700024694656/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762739 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762714.999338:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762714.999340:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789762714.999343:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e53800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024694656:192.168.203.107@tcp:38:kworker.0 00000100:00100000:0.0:1789762714.999347:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024694784:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789762714.999378:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789762714.999387:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762714/real 1789762714] req@ffffa06c07e51c00 x1876700024694784/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762739 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762714.999395:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51c00 x1876700024694784/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762739 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762714.999400:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762714.999403:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789762714.999405:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024694784:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789762715.254975:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762715.510944:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762716.278976:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762716.534994:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762717.303122:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762717.559005:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762718.327089:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762718.582984:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762719.351107:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762719.606933:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789762720.119015:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762720.119027:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789762720.119032:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762720.119044:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2153 00000100:00080000:2.0:1789762720.119057:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762720.119066:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789762720.119114:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e53480 x1876700024694912/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762720.119151:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762720.119156:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762720.119178:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762720.119184:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2153 00000100:00080000:2.0:1789762720.119190:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762720.119195:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789762720.119207:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762720.119226:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50a80 x1876700024695040/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762720.119245:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762720.119249:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762720.119251:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762720.119258:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2153 00000100:00080000:2.0:1789762720.119267:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1789762720.119269:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e53480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024694912:192.168.203.107@tcp:250:kworker.0 00000100:00080000:2.0:1789762720.119276:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789762720.119291:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762720.119304:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50700 x1876700024695168/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789762720.119403:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789762720.119458:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762720/real 1789762720] req@ffffa06c07e53480 x1876700024694912/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762745 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762720.119475:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e53480 x1876700024694912/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762745 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762720.119486:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762720.119489:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789762720.119507:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e53480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024694912:192.168.203.107@tcp:250:kworker.0 00000100:00100000:0.0:1789762720.119537:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024695040:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789762720.119557:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789762720.119566:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762720/real 1789762720] req@ffffa06c07e50a80 x1876700024695040/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762745 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762720.119575:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50a80 x1876700024695040/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762745 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762720.119581:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762720.119583:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789762720.119586:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024695040:192.168.203.107@tcp:38:kworker.0 00000100:00100000:0.0:1789762720.119590:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024695168:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789762720.119602:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789762720.119608:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762720/real 1789762720] req@ffffa06c07e50700 x1876700024695168/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762745 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762720.119615:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50700 x1876700024695168/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762745 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762720.119620:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762720.119622:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789762720.119624:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024695168:192.168.203.107@tcp:38:kworker.0 00010000:00010000:0.0:1789762720.375015:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762720.630945:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762721.399020:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762721.654995:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789762722.423025:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762722.678922:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789762723.447020:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762723.702977:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762724.470989:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762724.726937:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:1.0:1789762725.238971:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762725.238982:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789762725.238986:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762725.238999:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2159 00000100:00080000:1.0:1789762725.239011:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762725.239018:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789762725.239063:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2c000 x1876700024695296/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762725.239089:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762725.239093:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762725.239094:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762725.239117:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2159 00000100:00080000:1.0:1789762725.239121:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762725.239124:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789762725.239134:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024695296:192.168.203.107@tcp:250:kworker.0 00000008:00100000:1.0:1789762725.239135:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762725.239150:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2e300 x1876700024695424/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762725.239161:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762725.239164:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762725.239166:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762725.239168:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2159 00000100:00080000:1.0:1789762725.239170:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762725.239173:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789762725.239176:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762725.239184:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2ed80 x1876700024695552/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789762725.239221:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789762725.239264:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762725/real 1789762725] req@ffffa06c08e2c000 x1876700024695296/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762750 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762725.239302:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2c000 x1876700024695296/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762750 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762725.239311:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762725.239315:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789762725.239318:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024695296:192.168.203.107@tcp:250:kworker.0 00000100:00100000:0.0:1789762725.239341:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024695424:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789762725.239360:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789762725.239368:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762725/real 1789762725] req@ffffa06c08e2e300 x1876700024695424/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762750 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762725.239377:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2e300 x1876700024695424/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762750 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762725.239383:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762725.239386:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789762725.239388:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024695424:192.168.203.107@tcp:38:kworker.0 00000100:00100000:0.0:1789762725.239392:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024695552:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789762725.239403:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789762725.239415:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762725/real 1789762725] req@ffffa06c08e2ed80 x1876700024695552/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762750 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762725.239431:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2ed80 x1876700024695552/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762750 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762725.239437:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762725.239439:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789762725.239441:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024695552:192.168.203.107@tcp:38:kworker.0 00010000:00010000:0.0:1789762725.495007:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762725.750934:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789762726.519015:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762726.774970:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762727.542990:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762727.798956:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762728.567008:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762728.822973:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762729.590937:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762729.846926:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:1.0:1789762730.358952:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762730.358963:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789762730.358967:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762730.358977:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2164 00000100:00080000:1.0:1789762730.358988:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762730.359011:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789762730.359044:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2d500 x1876700024695680/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762730.359072:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762730.359077:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762730.359078:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762730.359083:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2164 00000100:00080000:1.0:1789762730.359087:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762730.359091:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789762730.359100:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762730.359114:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2ed80 x1876700024695808/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762730.359122:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024695680:192.168.203.107@tcp:250:kworker.0 00000100:00080000:1.0:1789762730.359133:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762730.359138:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762730.359140:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762730.359145:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2164 00000100:00080000:1.0:1789762730.359148:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762730.359153:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789762730.359158:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762730.359171:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2e300 x1876700024695936/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789762730.359181:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789762730.359216:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762730/real 1789762730] req@ffffa06c08e2d500 x1876700024695680/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762755 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762730.359230:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2d500 x1876700024695680/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762755 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762730.359238:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762730.359241:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789762730.359259:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024695680:192.168.203.107@tcp:250:kworker.0 00000100:00100000:0.0:1789762730.359278:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024695808:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789762730.359294:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789762730.359303:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762730/real 1789762730] req@ffffa06c08e2ed80 x1876700024695808/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762755 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762730.359311:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2ed80 x1876700024695808/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762755 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762730.359323:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762730.359326:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789762730.359328:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024695808:192.168.203.107@tcp:38:kworker.0 00000100:00100000:0.0:1789762730.359332:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024695936:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789762730.359345:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789762730.359352:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762730/real 1789762730] req@ffffa06c08e2e300 x1876700024695936/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762755 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762730.359359:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2e300 x1876700024695936/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762755 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762730.359364:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762730.359366:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789762730.359368:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024695936:192.168.203.107@tcp:38:kworker.0 00010000:00010000:2.0:1789762730.614970:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762730.871049:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762731.638984:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762731.895012:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762732.663042:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762732.919039:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789762733.687031:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762733.942960:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789762734.711135:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762734.966983:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:3.0:1789762735.479011:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762735.479028:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789762735.479042:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762735.479057:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2169 00000100:00080000:3.0:1789762735.479073:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762735.479082:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:3.0:1789762735.479134:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872e680 x1876700024696064/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762735.479162:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762735.479166:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762735.479168:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762735.479172:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2169 00000100:00080000:3.0:1789762735.479174:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762735.479178:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789762735.479189:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762735.479196:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024696064:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789762735.479248:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872ea00 x1876700024696192/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762735.479259:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762735.479263:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762735.479264:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762735.479267:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2169 00000100:00080000:3.0:1789762735.479270:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762735.479274:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789762735.479279:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762735.479288:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872f480 x1876700024696320/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789762735.479292:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789762735.479345:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762735/real 1789762735] req@ffffa06c1872e680 x1876700024696064/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762760 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762735.479361:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872e680 x1876700024696064/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762760 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762735.479370:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762735.479380:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789762735.479384:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024696064:192.168.203.107@tcp:250:kworker.0 00000100:00100000:0.0:1789762735.479409:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024696192:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789762735.479428:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789762735.479437:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762735/real 1789762735] req@ffffa06c1872ea00 x1876700024696192/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762760 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762735.479445:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872ea00 x1876700024696192/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762760 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762735.479452:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762735.479454:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789762735.479457:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024696192:192.168.203.107@tcp:38:kworker.0 00000100:00100000:0.0:1789762735.479460:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024696320:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789762735.479471:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789762735.479478:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762735/real 1789762735] req@ffffa06c1872f480 x1876700024696320/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762760 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762735.479485:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872f480 x1876700024696320/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762760 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762735.479490:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762735.479492:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789762735.479494:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024696320:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789762735.734995:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762735.990946:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789762736.759022:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762737.014960:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762737.783008:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762738.038930:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762738.806956:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762739.062933:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789762739.831005:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762740.086910:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789762740.598933:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762740.598957:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789762740.598960:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762740.598969:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2174 00000100:00080000:0.0:1789762740.598978:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762740.598983:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789762740.599011:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d3100 x1876700024696448/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762740.599034:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762740.599037:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762740.599039:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762740.599042:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2174 00000100:00080000:0.0:1789762740.599044:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762740.599046:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789762740.599054:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762740.599061:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0380 x1876700024696576/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762740.599069:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762740.599071:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762740.599079:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762740.599081:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2174 00000100:00080000:0.0:1789762740.599084:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762740.599086:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789762740.599090:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762740.599097:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d2680 x1876700024696704/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789762740.599119:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024696448:192.168.203.107@tcp:250:kworker.0 00000800:00000100:1.0:1789762740.599194:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762740.599229:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762740/real 1789762740] req@ffffa06c091d3100 x1876700024696448/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762765 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762740.599242:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d3100 x1876700024696448/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762765 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762740.599250:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762740.599253:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762740.599256:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024696448:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789762740.599261:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024696576:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789762740.599276:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762740.599283:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762740/real 1789762740] req@ffffa06c091d0380 x1876700024696576/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762765 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762740.599291:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0380 x1876700024696576/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762765 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762740.599307:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762740.599310:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762740.599312:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024696576:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789762740.599332:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024696704:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789762740.599346:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762740.599352:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762740/real 1789762740] req@ffffa06c091d2680 x1876700024696704/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762765 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762740.599359:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d2680 x1876700024696704/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762765 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762740.599364:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762740.599366:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762740.599368:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024696704:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789762740.854967:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762741.110920:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762741.878975:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762742.134965:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789762742.902972:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762743.158985:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762743.927058:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762744.182922:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762744.950951:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762745.206941:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:3.0:1789762745.718978:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762745.718993:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789762745.718998:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762745.719012:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2179 00000100:00080000:3.0:1789762745.719026:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762745.719037:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:3.0:1789762745.719083:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872fb80 x1876700024696832/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762745.719155:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762745.719160:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762745.719163:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762745.719170:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2179 00000100:00080000:3.0:1789762745.719176:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762745.719182:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789762745.719198:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762745.719203:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024696832:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789762745.719223:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872f480 x1876700024696960/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762745.719239:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762745.719244:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762745.719246:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762745.719252:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2179 00000100:00080000:3.0:1789762745.719257:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762745.719261:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789762745.719267:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1789762745.719279:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789762745.719320:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762745/real 1789762745] req@ffffa06c1872fb80 x1876700024696832/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762770 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762745.719335:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872fb80 x1876700024696832/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762770 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762745.719345:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762745.719349:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789762745.719352:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024696832:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789762745.719369:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872ea00 x1876700024697088/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762745.719377:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024696960:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789762745.719394:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789762745.719402:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762745/real 1789762745] req@ffffa06c1872f480 x1876700024696960/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762770 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762745.719411:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872f480 x1876700024696960/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762770 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762745.719426:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762745.719431:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789762745.719434:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024696960:192.168.203.107@tcp:38:kworker.0 00000100:00100000:0.0:1789762745.719570:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024697088:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789762745.719595:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789762745.719623:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762745/real 1789762745] req@ffffa06c1872ea00 x1876700024697088/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762770 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762745.719633:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872ea00 x1876700024697088/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762770 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762745.719639:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762745.719642:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789762745.719645:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024697088:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789762745.974973:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762746.231002:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762746.998960:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762747.254940:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762748.022953:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762748.278938:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762749.046990:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762749.302949:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762750.071019:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762750.326936:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:3.0:1789762750.838919:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762750.838933:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789762750.838939:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762750.838952:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2184 00000100:00080000:3.0:1789762750.838966:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762750.838977:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:3.0:1789762750.839032:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872e680 x1876700024697216/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762750.839065:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762750.839070:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762750.839073:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762750.839086:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2184 00000100:00080000:3.0:1789762750.839102:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762750.839108:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789762750.839121:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762750.839125:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024697216:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789762750.839143:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872ed80 x1876700024697344/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762750.839155:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762750.839158:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762750.839160:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762750.839163:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2184 00000100:00080000:3.0:1789762750.839168:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762750.839171:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789762750.839176:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762750.839198:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872dc00 x1876700024697472/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762750.839246:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024697344:192.168.203.107@tcp:38:kworker.0 00000100:00100000:0.0:1789762750.839274:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024697472:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789762751.095038:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762751.350940:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762752.118980:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762752.374946:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762753.142949:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762753.398983:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000400:00000100:1.0:1789762753.911242:0:2331:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.107:988 00000400:00000100:1.0:1789762753.911316:0:2331:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.107@tcp at host 192.168.203.107 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1789762753.911340:0:2331:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.107@tcp 00000800:00000100:1.0:1789762753.911390:0:2331:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.107@tcp 00000800:00000100:1.0:1789762753.911401:0:2331:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.107@tcp 00000100:00100000:0.0:1789762753.911452:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762750/real 1789762753] req@ffffa06c1872ed80 x1876700024697344/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762775 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762753.911511:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872ed80 x1876700024697344/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762775 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762753.911532:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762753.911541:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789762753.911550:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024697344:192.168.203.107@tcp:38:kworker.0 00000100:00100000:0.0:1789762753.911579:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762750/real 1789762753] req@ffffa06c1872dc00 x1876700024697472/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762775 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762753.911602:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872dc00 x1876700024697472/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762775 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762753.911633:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762753.911640:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789762753.911647:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024697472:192.168.203.107@tcp:38:kworker.0 00000100:00100000:0.0:1789762753.911659:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762750/real 1789762753] req@ffffa06c1872e680 x1876700024697216/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762775 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762753.911683:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872e680 x1876700024697216/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762775 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762753.911697:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762753.911703:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789762753.911709:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024697216:192.168.203.107@tcp:250:kworker.0 00010000:00010000:0.0:1789762754.167039:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762754.422962:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762755.190985:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762755.446999:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:1.0:1789762755.959008:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762755.959023:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789762755.959028:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762755.959042:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2189 00000100:00080000:1.0:1789762755.959069:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762755.959078:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789762755.959127:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2c000 x1876700024697600/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762755.959165:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762755.959169:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762755.959171:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762755.959177:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2189 00000100:00080000:1.0:1789762755.959182:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762755.959187:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789762755.959200:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762755.959217:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2c700 x1876700024697728/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762755.959227:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024697600:192.168.203.107@tcp:250:kworker.0 00000100:00080000:1.0:1789762755.959262:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762755.959266:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762755.959269:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762755.959274:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2189 00000100:00080000:1.0:1789762755.959279:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762755.959285:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789762755.959293:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762755.959311:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2ca80 x1876700024697856/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789762755.959325:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789762755.959375:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762755/real 1789762755] req@ffffa06c08e2c000 x1876700024697600/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762780 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762755.959402:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2c000 x1876700024697600/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762780 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762755.959414:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762755.959420:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789762755.959424:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024697600:192.168.203.107@tcp:250:kworker.0 00000100:00100000:0.0:1789762755.959454:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024697728:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789762755.959491:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789762755.959507:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762755/real 1789762755] req@ffffa06c08e2c700 x1876700024697728/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762780 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762755.959520:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2c700 x1876700024697728/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762780 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762755.959529:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762755.959533:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789762755.959537:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024697728:192.168.203.107@tcp:38:kworker.0 00000100:00100000:0.0:1789762755.959544:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024697856:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789762755.959566:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789762755.959578:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762755/real 1789762755] req@ffffa06c08e2ca80 x1876700024697856/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762780 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762755.959589:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2ca80 x1876700024697856/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762780 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762755.959597:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762755.959600:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789762755.959603:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024697856:192.168.203.107@tcp:38:kworker.0 00010000:00010000:2.0:1789762756.214985:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762756.470975:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789762757.239016:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762757.494964:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762758.263020:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762758.518930:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789762759.287070:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762759.542948:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762760.310984:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762760.566946:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:1.0:1789762761.078945:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762761.078961:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789762761.078966:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762761.078980:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2194 00000100:00080000:1.0:1789762761.078995:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762761.079005:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789762761.079072:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2d180 x1876700024697984/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762761.079105:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762761.079110:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762761.079134:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762761.079139:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2194 00000100:00080000:1.0:1789762761.079144:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762761.079149:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789762761.079163:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762761.079191:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2ca80 x1876700024698112/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762761.079206:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762761.079210:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762761.079212:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762761.079216:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2194 00000100:00080000:1.0:1789762761.079220:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762761.079224:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789762761.079231:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762761.079248:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2c700 x1876700024698240/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762761.079250:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024697984:192.168.203.107@tcp:250:kworker.0 00000800:00000100:0.0:1789762761.079367:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789762761.079431:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762761/real 1789762761] req@ffffa06c08e2d180 x1876700024697984/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762786 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762761.079493:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2d180 x1876700024697984/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762786 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762761.079516:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762761.079524:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789762761.079530:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024697984:192.168.203.107@tcp:250:kworker.0 00000100:00100000:0.0:1789762761.079543:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024698112:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789762761.079579:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789762761.079598:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762761/real 1789762761] req@ffffa06c08e2ca80 x1876700024698112/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762786 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762761.079613:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2ca80 x1876700024698112/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762786 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762761.079624:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762761.079628:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789762761.079633:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024698112:192.168.203.107@tcp:38:kworker.0 00000100:00100000:0.0:1789762761.079663:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024698240:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789762761.079684:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789762761.079697:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762761/real 1789762761] req@ffffa06c08e2c700 x1876700024698240/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762786 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762761.079711:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2c700 x1876700024698240/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762786 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762761.079739:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762761.079744:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789762761.079748:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024698240:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789762761.335053:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762761.590972:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762762.359062:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762762.614954:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762763.383033:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762763.638982:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762764.407006:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762764.663027:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762765.431103:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762765.686968:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:3.0:1789762766.199027:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762766.199040:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789762766.199045:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762766.199057:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2200 00000100:00080000:3.0:1789762766.199071:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762766.199079:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:3.0:1789762766.199129:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872c700 x1876700024698368/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762766.199157:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762766.199161:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762766.199162:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762766.199166:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2200 00000100:00080000:3.0:1789762766.199168:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762766.199171:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789762766.199183:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762766.199198:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872f100 x1876700024698496/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762766.199216:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024698368:192.168.203.107@tcp:250:kworker.0 00000100:00080000:3.0:1789762766.199251:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762766.199256:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762766.199258:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762766.199264:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2200 00000100:00080000:3.0:1789762766.199276:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762766.199282:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789762766.199289:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762766.199303:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872ca80 x1876700024698624/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789762766.199321:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789762766.199378:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762766/real 1789762766] req@ffffa06c1872c700 x1876700024698368/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762791 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762766.199400:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872c700 x1876700024698368/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762791 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762766.199413:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762766.199418:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789762766.199422:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024698368:192.168.203.107@tcp:250:kworker.0 00000100:00100000:0.0:1789762766.199455:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024698496:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789762766.199483:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789762766.199498:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762766/real 1789762766] req@ffffa06c1872f100 x1876700024698496/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762791 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762766.199512:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872f100 x1876700024698496/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762791 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762766.199521:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762766.199526:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789762766.199529:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024698496:192.168.203.107@tcp:38:kworker.0 00000100:00100000:0.0:1789762766.199549:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024698624:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789762766.199576:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789762766.199589:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762766/real 1789762766] req@ffffa06c1872ca80 x1876700024698624/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762791 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762766.199603:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872ca80 x1876700024698624/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762791 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762766.199611:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762766.199624:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789762766.199629:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024698624:192.168.203.107@tcp:38:kworker.0 00010000:00010000:0.0:1789762766.455040:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762766.710925:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762767.479039:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789762767.735004:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789762768.503107:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762768.758951:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762769.527016:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762769.782936:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762770.551001:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762770.806963:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:1.0:1789762771.319060:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762771.319072:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789762771.319095:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762771.319107:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2205 00000100:00080000:1.0:1789762771.319120:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762771.319128:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789762771.319168:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2c000 x1876700024698752/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762771.319206:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762771.319211:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762771.319213:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762771.319217:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2205 00000100:00080000:1.0:1789762771.319221:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762771.319224:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789762771.319235:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762771.319249:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2e300 x1876700024698880/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762771.319266:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762771.319270:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762771.319272:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1789762771.319280:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024698752:192.168.203.107@tcp:250:kworker.0 00000100:00080000:1.0:1789762771.319282:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2205 00000100:00080000:1.0:1789762771.319292:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762771.319298:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789762771.319303:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762771.319317:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2ed80 x1876700024699008/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789762771.319380:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789762771.319435:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762771/real 1789762771] req@ffffa06c08e2c000 x1876700024698752/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762796 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762771.319466:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2c000 x1876700024698752/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762796 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762771.319479:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762771.319483:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789762771.319505:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024698752:192.168.203.107@tcp:250:kworker.0 00000100:00100000:0.0:1789762771.319534:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024698880:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789762771.319584:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789762771.319607:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762771/real 1789762771] req@ffffa06c08e2e300 x1876700024698880/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762796 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762771.319632:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2e300 x1876700024698880/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762796 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762771.319649:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762771.319656:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789762771.319663:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024698880:192.168.203.107@tcp:38:kworker.0 00000100:00100000:0.0:1789762771.319675:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024699008:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789762771.319706:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789762771.319737:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762771/real 1789762771] req@ffffa06c08e2ed80 x1876700024699008/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762796 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762771.319753:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2ed80 x1876700024699008/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762796 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762771.319763:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762771.319767:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789762771.319771:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024699008:192.168.203.107@tcp:38:kworker.0 00010000:00010000:2.0:1789762771.575074:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762771.831034:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762772.599028:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762772.854964:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789762773.623028:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762773.879010:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762774.647013:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762774.903007:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762775.670980:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762775.926970:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789762776.439037:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762776.439051:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789762776.439056:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762776.439068:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2210 00000100:00080000:0.0:1789762776.439109:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762776.439119:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789762776.439164:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d2a00 x1876700024699136/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762776.439202:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762776.439206:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762776.439208:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762776.439213:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2210 00000100:00080000:0.0:1789762776.439226:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762776.439230:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789762776.439243:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762776.439258:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0e00 x1876700024699264/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789762776.439258:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024699136:192.168.203.107@tcp:250:kworker.0 00000100:00080000:0.0:1789762776.439288:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762776.439292:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762776.439295:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762776.439302:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2210 00000100:00080000:0.0:1789762776.439305:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762776.439309:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789762776.439315:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762776.439326:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d1500 x1876700024699392/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789762776.439343:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762776.439395:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762776/real 1789762776] req@ffffa06c091d2a00 x1876700024699136/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762801 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762776.439415:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d2a00 x1876700024699136/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762801 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762776.439424:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762776.439427:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762776.439430:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024699136:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789762776.439453:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024699264:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789762776.439475:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762776.439488:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762776/real 1789762776] req@ffffa06c091d0e00 x1876700024699264/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762801 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762776.439502:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0e00 x1876700024699264/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762801 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762776.439512:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762776.439516:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762776.439520:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024699264:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789762776.439526:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024699392:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789762776.439550:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762776.439571:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762776/real 1789762776] req@ffffa06c091d1500 x1876700024699392/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762801 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762776.439580:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d1500 x1876700024699392/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762801 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762776.439585:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762776.439588:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762776.439590:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024699392:192.168.203.107@tcp:38:kworker.0 00010000:00010000:0.0:1789762776.695003:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762776.950888:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762777.718993:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762777.974985:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762778.742981:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762778.998948:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789762779.767073:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762780.022944:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762780.790957:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762781.046985:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789762781.559092:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762781.559107:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789762781.559115:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762781.559134:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2215 00000100:00080000:2.0:1789762781.559152:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762781.559166:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789762781.559224:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e52d80 x1876700024699520/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762781.559272:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762781.559280:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762781.559285:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762781.559295:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2215 00000100:00080000:2.0:1789762781.559306:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762781.559316:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789762781.559345:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762781.559352:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e52d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024699520:192.168.203.107@tcp:250:kworker.0 00000100:00100000:2.0:1789762781.559374:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e53100 x1876700024699648/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762781.559395:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762781.559400:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762781.559403:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762781.559409:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2215 00000100:00080000:2.0:1789762781.559428:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762781.559435:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789762781.559442:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1789762781.559448:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789762781.559469:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e52a00 x1876700024699776/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762781.559525:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762781/real 1789762781] req@ffffa06c07e52d80 x1876700024699520/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762806 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762781.559555:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e52d80 x1876700024699520/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762806 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762781.559571:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762781.559577:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789762781.559582:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e52d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024699520:192.168.203.107@tcp:250:kworker.0 00000100:00100000:0.0:1789762781.559616:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e53100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024699648:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789762781.559678:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789762781.559705:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762781/real 1789762781] req@ffffa06c07e53100 x1876700024699648/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762806 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762781.559734:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e53100 x1876700024699648/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762806 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762781.559746:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762781.559752:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789762781.559757:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e53100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024699648:192.168.203.107@tcp:38:kworker.0 00000100:00100000:0.0:1789762781.559766:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e52a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024699776:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789762781.559804:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789762781.559826:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762781/real 1789762781] req@ffffa06c07e52a00 x1876700024699776/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762806 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762781.559955:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e52a00 x1876700024699776/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762806 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762781.559983:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762781.559994:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789762781.560005:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e52a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024699776:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789762781.814969:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762782.070955:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789762782.839069:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762783.094939:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762783.863024:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762784.118920:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762784.886938:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762785.143003:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789762785.911303:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762786.166978:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789762786.678971:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762786.678981:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789762786.678986:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762786.678995:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2220 00000100:00080000:2.0:1789762786.679006:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762786.679013:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789762786.679051:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e52a00 x1876700024699904/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762786.679082:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762786.679086:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762786.679089:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762786.679095:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2220 00000100:00080000:2.0:1789762786.679100:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762786.679105:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789762786.679122:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762786.679133:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e52a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024699904:192.168.203.107@tcp:250:kworker.0 00000100:00100000:2.0:1789762786.679163:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e53100 x1876700024700032/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762786.679179:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762786.679183:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762786.679186:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762786.679191:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2220 00000100:00080000:2.0:1789762786.679196:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762786.679201:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789762786.679208:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1789762786.679222:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789762786.679224:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e52d80 x1876700024700160/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762786.679281:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762786/real 1789762786] req@ffffa06c07e52a00 x1876700024699904/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762811 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762786.679301:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e52a00 x1876700024699904/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762811 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762786.679311:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762786.679315:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789762786.679319:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e52a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024699904:192.168.203.107@tcp:250:kworker.0 00000100:00100000:0.0:1789762786.679350:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e53100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024700032:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789762786.679379:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789762786.679393:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762786/real 1789762786] req@ffffa06c07e53100 x1876700024700032/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762811 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762786.679408:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e53100 x1876700024700032/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762811 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762786.679438:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762786.679442:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789762786.679445:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e53100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024700032:192.168.203.107@tcp:38:kworker.0 00000100:00100000:0.0:1789762786.679451:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e52d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024700160:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789762786.679470:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789762786.679480:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762786/real 1789762786] req@ffffa06c07e52d80 x1876700024700160/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762811 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762786.679488:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e52d80 x1876700024700160/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762811 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762786.679494:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762786.679496:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789762786.679499:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e52d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024700160:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789762786.935007:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762787.191010:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762787.958939:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762788.214938:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762788.983071:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762789.238946:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762790.007038:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762790.262968:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762791.031055:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762791.286947:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:1.0:1789762791.798991:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762791.799001:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789762791.799005:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762791.799014:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2225 00000100:00080000:1.0:1789762791.799025:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762791.799032:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789762791.799067:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2d500 x1876700024700288/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762791.799094:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762791.799097:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762791.799099:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762791.799104:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2225 00000100:00080000:1.0:1789762791.799107:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762791.799111:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789762791.799120:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762791.799153:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2ed80 x1876700024700416/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762791.799168:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762791.799172:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762791.799175:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1789762791.799182:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024700288:192.168.203.107@tcp:250:kworker.0 00000100:00080000:1.0:1789762791.799183:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2225 00000100:00080000:1.0:1789762791.799200:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762791.799207:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789762791.799214:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762791.799231:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2e300 x1876700024700544/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789762791.799281:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789762791.799338:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762791/real 1789762791] req@ffffa06c08e2d500 x1876700024700288/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762816 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762791.799355:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2d500 x1876700024700288/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762816 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762791.799364:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762791.799368:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789762791.799384:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024700288:192.168.203.107@tcp:250:kworker.0 00000100:00100000:0.0:1789762791.799411:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024700416:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789762791.799430:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789762791.799439:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762791/real 1789762791] req@ffffa06c08e2ed80 x1876700024700416/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762816 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762791.799447:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2ed80 x1876700024700416/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762816 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762791.799453:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762791.799455:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789762791.799458:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024700416:192.168.203.107@tcp:38:kworker.0 00000100:00100000:0.0:1789762791.799462:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024700544:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789762791.799473:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789762791.799479:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762791/real 1789762791] req@ffffa06c08e2e300 x1876700024700544/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762816 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762791.799486:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2e300 x1876700024700544/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762816 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762791.799492:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762791.799494:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789762791.799498:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024700544:192.168.203.107@tcp:38:kworker.0 00010000:00010000:0.0:1789762792.055022:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762792.310942:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762793.078995:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762793.334924:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762794.102981:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762794.358934:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762795.126956:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762795.383014:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762796.150954:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762796.406959:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789762796.918961:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762796.918972:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789762796.918976:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762796.918986:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2230 00000100:00080000:2.0:1789762796.918997:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762796.919020:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789762796.919056:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50700 x1876700024700672/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762796.919086:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762796.919090:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762796.919092:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762796.919097:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2230 00000100:00080000:2.0:1789762796.919102:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762796.919107:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789762796.919117:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762796.919135:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50a80 x1876700024700800/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762796.919143:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024700672:192.168.203.107@tcp:250:kworker.0 00000100:00080000:2.0:1789762796.919149:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762796.919155:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762796.919159:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762796.919164:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2230 00000100:00080000:2.0:1789762796.919168:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762796.919173:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789762796.919179:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762796.919194:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e53480 x1876700024700928/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789762796.919223:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789762796.919260:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762796/real 1789762796] req@ffffa06c07e50700 x1876700024700672/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762821 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762796.919279:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50700 x1876700024700672/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762821 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762796.919290:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762796.919294:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789762796.919298:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024700672:192.168.203.107@tcp:250:kworker.0 00000100:00100000:0.0:1789762796.919322:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024700800:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789762796.919346:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789762796.919360:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762796/real 1789762796] req@ffffa06c07e50a80 x1876700024700800/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762821 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762796.919375:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50a80 x1876700024700800/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762821 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762796.919385:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762796.919388:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789762796.919392:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024700800:192.168.203.107@tcp:38:kworker.0 00000100:00100000:0.0:1789762796.919399:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e53480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024700928:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789762796.919418:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789762796.919428:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762796/real 1789762796] req@ffffa06c07e53480 x1876700024700928/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762821 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762796.919455:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e53480 x1876700024700928/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762821 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762796.919464:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762796.919467:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789762796.919471:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e53480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024700928:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789762797.174959:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762797.430917:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762798.198957:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762798.454950:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762799.222999:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762799.478920:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762800.247058:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762800.502928:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762801.270951:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762801.526960:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:1.0:1789762802.039021:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762802.039035:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789762802.039040:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762802.039055:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2235 00000100:00080000:1.0:1789762802.039069:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762802.039077:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789762802.039125:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2c000 x1876700024701056/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762802.039154:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762802.039157:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762802.039159:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762802.039162:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2235 00000100:00080000:1.0:1789762802.039165:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762802.039185:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789762802.039196:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762802.039207:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2c700 x1876700024701184/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762802.039220:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762802.039223:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762802.039225:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762802.039231:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2235 00000100:00100000:0.0:1789762802.039234:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024701056:192.168.203.107@tcp:250:kworker.0 00000100:00080000:1.0:1789762802.039236:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762802.039243:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789762802.039252:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762802.039262:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2ca80 x1876700024701312/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789762802.039311:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789762802.039348:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762802/real 1789762802] req@ffffa06c08e2c000 x1876700024701056/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762827 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762802.039362:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2c000 x1876700024701056/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762827 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762802.039371:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762802.039374:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789762802.039377:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024701056:192.168.203.107@tcp:250:kworker.0 00000100:00100000:0.0:1789762802.039396:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024701184:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789762802.039410:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789762802.039418:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762802/real 1789762802] req@ffffa06c08e2c700 x1876700024701184/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762827 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762802.039426:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2c700 x1876700024701184/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762827 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762802.039448:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762802.039451:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789762802.039453:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024701184:192.168.203.107@tcp:38:kworker.0 00000100:00100000:0.0:1789762802.039458:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024701312:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789762802.039477:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789762802.039488:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762802/real 1789762802] req@ffffa06c08e2ca80 x1876700024701312/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762827 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762802.039512:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2ca80 x1876700024701312/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762827 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762802.039521:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762802.039525:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789762802.039528:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024701312:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789762802.295008:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762802.550914:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762803.318974:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762803.574923:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762804.342962:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762804.599013:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762805.367027:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762805.622934:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762806.391019:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762806.646982:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789762807.159008:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762807.159017:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789762807.159019:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762807.159027:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2240 00000100:00080000:0.0:1789762807.159035:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762807.159041:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789762807.159073:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0000 x1876700024701440/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762807.159098:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762807.159114:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762807.159116:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762807.159119:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2240 00000100:00080000:0.0:1789762807.159123:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762807.159126:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789762807.159134:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762807.159145:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d3b80 x1876700024701568/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789762807.159146:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024701440:192.168.203.107@tcp:250:kworker.0 00000100:00080000:0.0:1789762807.159165:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762807.159168:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762807.159171:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762807.159175:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2240 00000100:00080000:0.0:1789762807.159178:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762807.159183:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789762807.159188:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762807.159201:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d2300 x1876700024701696/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789762807.159217:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762807.159250:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762807/real 1789762807] req@ffffa06c091d0000 x1876700024701440/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762832 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762807.159263:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0000 x1876700024701440/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762832 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762807.159271:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762807.159274:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762807.159277:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024701440:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789762807.159311:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024701568:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789762807.159328:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762807.159337:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762807/real 1789762807] req@ffffa06c091d3b80 x1876700024701568/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762832 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762807.159345:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d3b80 x1876700024701568/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762832 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762807.159351:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762807.159353:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762807.159356:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024701568:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789762807.159359:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024701696:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789762807.159369:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762807.159375:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762807/real 1789762807] req@ffffa06c091d2300 x1876700024701696/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762832 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762807.159382:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d2300 x1876700024701696/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762832 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762807.159388:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762807.159389:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762807.159391:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024701696:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789762807.414998:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762807.670973:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762808.439021:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762808.694978:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762809.462981:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762809.718945:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762810.486997:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762810.742971:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789762811.511007:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762811.766947:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789762812.278934:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762812.278945:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789762812.278949:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762812.278960:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2246 00000100:00080000:2.0:1789762812.278972:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762812.278979:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789762812.279026:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51c00 x1876700024701824/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762812.279051:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762812.279054:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762812.279056:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762812.279059:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2246 00000100:00080000:2.0:1789762812.279079:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762812.279083:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789762812.279095:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762812.279112:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e53800 x1876700024701952/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789762812.279119:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024701824:192.168.203.107@tcp:250:kworker.0 00000100:00080000:2.0:1789762812.279133:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762812.279138:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762812.279140:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762812.279144:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2246 00000100:00080000:2.0:1789762812.279149:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762812.279152:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789762812.279157:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762812.279168:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e52680 x1876700024702080/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789762812.279208:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762812.279265:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762812/real 1789762812] req@ffffa06c07e51c00 x1876700024701824/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762837 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762812.279286:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51c00 x1876700024701824/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762837 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762812.279298:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762812.279304:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762812.279309:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024701824:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789762812.279338:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e53800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024701952:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789762812.279363:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762812.279376:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762812/real 1789762812] req@ffffa06c07e53800 x1876700024701952/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762837 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762812.279389:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e53800 x1876700024701952/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762837 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762812.279399:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762812.279402:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762812.279405:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e53800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024701952:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789762812.279412:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e52680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024702080:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789762812.279432:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762812.279444:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762812/real 1789762812] req@ffffa06c07e52680 x1876700024702080/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762837 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762812.279455:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e52680 x1876700024702080/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762837 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762812.279476:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762812.279480:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762812.279483:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e52680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024702080:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789762812.534978:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762812.790965:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762813.559016:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762813.814959:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789762814.582990:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762814.838979:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762815.606972:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762815.862932:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762816.631043:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762816.886944:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789762817.398946:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762817.398955:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789762817.398960:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762817.398971:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2251 00000100:00080000:2.0:1789762817.398983:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762817.398990:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789762817.399049:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51f80 x1876700024702208/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762817.399076:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762817.399081:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762817.399083:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762817.399089:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2251 00000100:00080000:2.0:1789762817.399097:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762817.399104:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789762817.399118:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762817.399134:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024702208:192.168.203.107@tcp:250:kworker.0 00000100:00100000:2.0:1789762817.399139:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50380 x1876700024702336/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762817.399159:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762817.399163:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762817.399165:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762817.399167:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2251 00000100:00080000:2.0:1789762817.399173:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762817.399176:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789762817.399180:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762817.399192:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e53b80 x1876700024702464/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789762817.399312:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024702336:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789762817.399338:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e53b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024702464:192.168.203.107@tcp:38:kworker.0 00010000:00010000:2.0:1789762817.654989:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762817.910931:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762818.678988:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762818.934971:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789762819.703013:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762819.958926:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000400:00000100:0.0:1789762820.472038:0:2328:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.107:988 00000400:00000100:0.0:1789762820.472092:0:2328:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.107@tcp at host 192.168.203.107 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1789762820.472109:0:2328:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.107@tcp 00000800:00000100:0.0:1789762820.472159:0:2328:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.107@tcp 00000800:00000100:0.0:1789762820.472170:0:2328:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.107@tcp 00000100:00100000:3.0:1789762820.472194:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762817/real 1789762820] req@ffffa06c07e50380 x1876700024702336/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762842 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762820.472237:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50380 x1876700024702336/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762842 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762820.472251:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762820.472256:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789762820.472261:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024702336:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789762820.472271:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762817/real 1789762820] req@ffffa06c07e53b80 x1876700024702464/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762842 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762820.472285:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e53b80 x1876700024702464/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762842 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762820.472293:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762820.472297:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789762820.472301:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e53b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024702464:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789762820.472308:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762817/real 1789762820] req@ffffa06c07e51f80 x1876700024702208/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762842 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762820.472321:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51f80 x1876700024702208/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762842 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762820.472329:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762820.472332:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789762820.472347:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024702208:192.168.203.107@tcp:250:kworker.0 00010000:00010000:2.0:1789762820.727007:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762820.982941:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762821.751022:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762822.006986:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789762822.518938:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762822.518956:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789762822.518961:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762822.518977:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2256 00000100:00080000:2.0:1789762822.518993:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762822.519004:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789762822.519062:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50e00 x1876700024702592/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762822.519095:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762822.519100:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762822.519102:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762822.519107:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2256 00000100:00080000:2.0:1789762822.519113:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762822.519118:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789762822.519130:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762822.519136:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024702592:192.168.203.107@tcp:250:kworker.0 00000100:00100000:2.0:1789762822.519150:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51500 x1876700024702720/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762822.519165:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762822.519170:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762822.519171:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762822.519176:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2256 00000100:00080000:2.0:1789762822.519181:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762822.519205:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789762822.519212:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762822.519226:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50000 x1876700024702848/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789762822.519245:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789762822.519316:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762822/real 1789762822] req@ffffa06c07e50e00 x1876700024702592/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762847 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762822.519338:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50e00 x1876700024702592/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762847 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762822.519351:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762822.519356:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789762822.519361:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024702592:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789762822.519390:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024702720:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789762822.519431:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789762822.519446:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762822/real 1789762822] req@ffffa06c07e51500 x1876700024702720/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762847 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762822.519460:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51500 x1876700024702720/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762847 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762822.519470:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762822.519474:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789762822.519478:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024702720:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789762822.519484:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024702848:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789762822.519503:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789762822.519523:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762822/real 1789762822] req@ffffa06c07e50000 x1876700024702848/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762847 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762822.519566:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50000 x1876700024702848/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762847 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762822.519576:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762822.519580:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789762822.519584:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024702848:192.168.203.107@tcp:38:kworker.0 00010000:00010000:0.0:1789762822.775040:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762823.030964:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762823.799154:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762824.054996:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789762824.822999:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762825.078974:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789762825.846997:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762826.102986:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762826.870982:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762827.126945:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:1.0:1789762827.639026:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762827.639037:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789762827.639041:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762827.639051:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2261 00000100:00080000:1.0:1789762827.639061:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762827.639068:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789762827.639105:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2d180 x1876700024702976/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762827.639139:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762827.639143:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762827.639146:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762827.639150:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2261 00000100:00080000:1.0:1789762827.639155:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762827.639159:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789762827.639169:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762827.639186:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2ca80 x1876700024703104/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789762827.639196:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024702976:192.168.203.107@tcp:250:kworker.0 00000100:00080000:1.0:1789762827.639203:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762827.639221:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762827.639223:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762827.639229:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2261 00000100:00080000:1.0:1789762827.639234:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762827.639239:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789762827.639245:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1789762827.639262:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762827.639263:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2c700 x1876700024703232/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789762827.639304:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762827/real 1789762827] req@ffffa06c08e2d180 x1876700024702976/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762852 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762827.639319:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2d180 x1876700024702976/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762852 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762827.639327:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762827.639330:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789762827.639333:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024702976:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789762827.639350:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024703104:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789762827.639365:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789762827.639373:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762827/real 1789762827] req@ffffa06c08e2ca80 x1876700024703104/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762852 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762827.639381:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2ca80 x1876700024703104/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762852 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762827.639399:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762827.639401:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789762827.639404:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024703104:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789762827.639408:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024703232:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789762827.639422:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789762827.639433:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762827/real 1789762827] req@ffffa06c08e2c700 x1876700024703232/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762852 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762827.639445:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2c700 x1876700024703232/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762852 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762827.639454:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762827.639458:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789762827.639461:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024703232:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789762827.895004:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762828.151037:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789762828.919027:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762829.174932:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762829.942983:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762830.198957:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762830.967011:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762831.222922:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789762831.991019:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762832.246986:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789762832.758978:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762832.758989:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789762832.758993:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762832.759004:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2266 00000100:00080000:2.0:1789762832.759016:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762832.759024:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789762832.759068:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51180 x1876700024703360/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762832.759097:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762832.759100:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762832.759101:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762832.759105:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2266 00000100:00080000:2.0:1789762832.759129:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762832.759132:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789762832.759142:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762832.759153:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e52300 x1876700024703488/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762832.759165:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762832.759168:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762832.759169:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762832.759172:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2266 00000100:00080000:2.0:1789762832.759174:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762832.759176:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789762832.759181:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762832.759188:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51880 x1876700024703616/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789762832.759201:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024703360:192.168.203.107@tcp:250:kworker.0 00000800:00000100:3.0:1789762832.759270:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789762832.759310:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762832/real 1789762832] req@ffffa06c07e51180 x1876700024703360/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762857 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762832.759325:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51180 x1876700024703360/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762857 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762832.759351:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762832.759354:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789762832.759357:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024703360:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789762832.759362:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e52300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024703488:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789762832.759380:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789762832.759389:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762832/real 1789762832] req@ffffa06c07e52300 x1876700024703488/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762857 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762832.759397:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e52300 x1876700024703488/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762857 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762832.759402:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762832.759405:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789762832.759407:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e52300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024703488:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789762832.759425:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024703616:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789762832.759435:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789762832.759441:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762832/real 1789762832] req@ffffa06c07e51880 x1876700024703616/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762857 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762832.759448:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51880 x1876700024703616/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762857 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762832.759453:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762832.759455:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789762832.759457:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024703616:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789762833.014998:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762833.270939:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762834.039023:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762834.294950:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789762835.062958:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762835.318977:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762836.086995:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762836.342957:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789762837.110987:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762837.366942:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:1.0:1789762837.878950:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762837.878964:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789762837.878969:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762837.878981:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2271 00000100:00080000:1.0:1789762837.878996:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762837.879004:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789762837.879053:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2c000 x1876700024703744/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762837.879089:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762837.879093:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762837.879095:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762837.879100:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2271 00000100:00080000:1.0:1789762837.879106:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762837.879129:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:3.0:1789762837.879142:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024703744:192.168.203.107@tcp:250:kworker.0 00000008:00100000:1.0:1789762837.879143:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762837.879168:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2e300 x1876700024703872/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762837.879183:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762837.879187:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762837.879189:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762837.879192:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2271 00000100:00080000:1.0:1789762837.879199:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762837.879202:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789762837.879206:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762837.879215:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2ed80 x1876700024704000/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789762837.879225:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789762837.879269:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762837/real 1789762837] req@ffffa06c08e2c000 x1876700024703744/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762862 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762837.879284:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2c000 x1876700024703744/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762862 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762837.879292:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762837.879295:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789762837.879298:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024703744:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789762837.879320:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024703872:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789762837.879334:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789762837.879342:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762837/real 1789762837] req@ffffa06c08e2e300 x1876700024703872/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762862 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762837.879350:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2e300 x1876700024703872/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762862 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762837.879355:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762837.879357:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789762837.879359:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024703872:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789762837.879363:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024704000:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789762837.879388:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789762837.879395:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762837/real 1789762837] req@ffffa06c08e2ed80 x1876700024704000/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762862 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762837.879402:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2ed80 x1876700024704000/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762862 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762837.879407:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762837.879409:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789762837.879411:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024704000:192.168.203.107@tcp:38:kworker.0 00010000:00010000:1.0:1789762838.134956:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762838.390993:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789762839.159002:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762839.414970:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762840.182991:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762840.438987:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789762841.207038:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762841.462926:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762842.230992:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762842.486944:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:1.0:1789762842.998919:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762842.998930:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789762842.998935:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762842.998945:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2276 00000100:00080000:1.0:1789762842.998958:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762842.998965:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789762842.999015:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2d500 x1876700024704128/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762842.999071:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762842.999075:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762842.999076:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762842.999080:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2276 00000100:00080000:1.0:1789762842.999083:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762842.999087:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789762842.999097:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762842.999111:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2ed80 x1876700024704256/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762842.999121:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762842.999125:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:3.0:1789762842.999125:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024704128:192.168.203.107@tcp:250:kworker.0 00000100:00080000:1.0:1789762842.999129:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762842.999133:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2276 00000100:00080000:1.0:1789762842.999136:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762842.999139:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789762842.999143:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762842.999151:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2e300 x1876700024704384/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789762842.999208:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789762842.999259:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762842/real 1789762842] req@ffffa06c08e2d500 x1876700024704128/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762867 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762842.999277:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2d500 x1876700024704128/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762867 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762842.999286:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762842.999290:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789762842.999292:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024704128:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789762842.999315:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024704256:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789762842.999332:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789762842.999341:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762842/real 1789762842] req@ffffa06c08e2ed80 x1876700024704256/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762867 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762842.999361:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2ed80 x1876700024704256/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762867 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762842.999368:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762842.999370:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789762842.999373:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024704256:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789762842.999376:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024704384:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789762842.999391:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789762842.999398:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762842/real 1789762842] req@ffffa06c08e2e300 x1876700024704384/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762867 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762842.999420:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2e300 x1876700024704384/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762867 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762842.999425:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762842.999428:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789762842.999430:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024704384:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789762843.254977:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762843.510987:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789762844.278959:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762844.534949:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762845.303002:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789762845.558996:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762846.327022:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789762846.582916:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789762847.350975:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789762847.606938:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789762848.118927:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762848.118936:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789762848.118939:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762848.118946:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2281 00000100:00080000:0.0:1789762848.118956:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762848.118961:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789762848.118996:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d2300 x1876700024704512/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762848.119024:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762848.119034:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762848.119036:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762848.119040:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2281 00000100:00080000:0.0:1789762848.119044:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762848.119048:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789762848.119056:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762848.119067:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024704512:192.168.203.107@tcp:250:kworker.0 00000100:00100000:0.0:1789762848.119069:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d3b80 x1876700024704640/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762848.119083:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762848.119085:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762848.119087:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762848.119103:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2281 00000100:00080000:0.0:1789762848.119106:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762848.119110:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789762848.119114:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762848.119123:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0000 x1876700024704768/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789762848.119128:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789762848.119167:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762848/real 1789762848] req@ffffa06c091d2300 x1876700024704512/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762873 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762848.119181:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d2300 x1876700024704512/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762873 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762848.119195:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762848.119198:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789762848.119201:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024704512:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789762848.119218:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024704640:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789762848.119235:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789762848.119243:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762848/real 1789762848] req@ffffa06c091d3b80 x1876700024704640/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762873 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762848.119251:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d3b80 x1876700024704640/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762873 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762848.119257:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762848.119259:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789762848.119261:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024704640:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789762848.119265:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024704768:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789762848.119274:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789762848.119280:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762848/real 1789762848] req@ffffa06c091d0000 x1876700024704768/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762873 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762848.119287:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0000 x1876700024704768/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762873 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762848.119292:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762848.119294:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789762848.119296:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024704768:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789762848.374952:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789762848.630905:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789762849.398958:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789762849.654952:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789762850.423010:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789762850.678923:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762851.446986:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789762851.702918:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762852.470958:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789762852.726905:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789762853.238973:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762853.238982:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789762853.238985:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762853.238993:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2287 00000100:00080000:0.0:1789762853.239002:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762853.239009:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789762853.239045:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d1500 x1876700024704896/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762853.239069:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762853.239073:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762853.239075:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762853.239082:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2287 00000100:00080000:0.0:1789762853.239086:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762853.239103:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789762853.239112:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024704896:192.168.203.107@tcp:250:kworker.0 00000008:00100000:0.0:1789762853.239113:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762853.239133:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0e00 x1876700024705024/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762853.239144:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762853.239148:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762853.239149:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762853.239154:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2287 00000100:00080000:0.0:1789762853.239158:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762853.239162:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789762853.239165:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762853.239178:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d2a00 x1876700024705152/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789762853.239226:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789762853.239283:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762853/real 1789762853] req@ffffa06c091d1500 x1876700024704896/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762878 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762853.239303:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d1500 x1876700024704896/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762878 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762853.239320:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762853.239339:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789762853.239343:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024704896:192.168.203.107@tcp:250:kworker.0 00000100:00100000:2.0:1789762853.239375:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024705024:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789762853.239404:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789762853.239420:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762853/real 1789762853] req@ffffa06c091d0e00 x1876700024705024/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762878 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762853.239434:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0e00 x1876700024705024/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762878 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762853.239444:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762853.239447:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789762853.239449:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024705024:192.168.203.107@tcp:38:kworker.0 00000100:00100000:2.0:1789762853.239456:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024705152:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789762853.239476:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789762853.239500:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762853/real 1789762853] req@ffffa06c091d2a00 x1876700024705152/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762878 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762853.239512:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d2a00 x1876700024705152/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762878 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762853.239518:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762853.239521:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789762853.239524:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024705152:192.168.203.107@tcp:38:kworker.0 00010000:00010000:0.0:1789762853.494947:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789762853.750946:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762854.518949:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789762854.774923:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762855.542948:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789762855.798989:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762856.567007:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789762856.822901:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762857.590944:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789762857.846927:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:1.0:1789762858.358921:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762858.358928:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789762858.358931:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762858.358954:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2292 00000100:00080000:1.0:1789762858.358962:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762858.358968:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789762858.358998:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2c000 x1876700024705280/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762858.359019:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762858.359022:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762858.359023:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762858.359026:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2292 00000100:00080000:1.0:1789762858.359028:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762858.359030:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789762858.359061:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024705280:192.168.203.107@tcp:250:kworker.0 00000008:00100000:1.0:1789762858.359082:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762858.359095:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2c700 x1876700024705408/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762858.359105:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762858.359108:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762858.359109:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762858.359111:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2292 00000100:00080000:1.0:1789762858.359120:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762858.359123:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000800:00000100:2.0:1789762858.359128:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000008:00100000:1.0:1789762858.359131:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762858.359141:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2ca80 x1876700024705536/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789762858.359166:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762858/real 1789762858] req@ffffa06c08e2c000 x1876700024705280/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762883 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762858.359180:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2c000 x1876700024705280/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762883 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762858.359189:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762858.359192:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789762858.359194:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024705280:192.168.203.107@tcp:250:kworker.0 00000100:00100000:2.0:1789762858.359231:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024705408:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789762858.359248:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789762858.359257:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762858/real 1789762858] req@ffffa06c08e2c700 x1876700024705408/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762883 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762858.359266:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2c700 x1876700024705408/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762883 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762858.359278:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762858.359281:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789762858.359294:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024705408:192.168.203.107@tcp:38:kworker.0 00000100:00100000:2.0:1789762858.359299:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024705536:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789762858.359315:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789762858.359323:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762858/real 1789762858] req@ffffa06c08e2ca80 x1876700024705536/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762883 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762858.359330:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2ca80 x1876700024705536/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762883 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762858.359336:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762858.359338:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789762858.359340:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024705536:192.168.203.107@tcp:38:kworker.0 00010000:00010000:2.0:1789762858.614980:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789762858.870921:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762859.638948:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762859.895009:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762860.662985:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762860.918914:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762861.686951:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762861.942934:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762862.710983:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762862.966969:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789762863.479054:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762863.479080:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789762863.479086:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762863.479102:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2297 00000100:00080000:0.0:1789762863.479119:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762863.479155:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789762863.479224:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d2680 x1876700024705664/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762863.479298:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762863.479306:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762863.479310:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762863.479319:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2297 00000100:00080000:0.0:1789762863.479327:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762863.479333:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789762863.479346:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762863.479368:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0380 x1876700024705792/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762863.479387:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1789762863.479397:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024705664:192.168.203.107@tcp:250:kworker.0 00000100:00080000:0.0:1789762863.479398:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762863.479403:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762863.479417:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2297 00000100:00080000:0.0:1789762863.479422:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762863.479426:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789762863.479431:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762863.479449:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d3100 x1876700024705920/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789762863.479507:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789762863.479575:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762863/real 1789762863] req@ffffa06c091d2680 x1876700024705664/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762888 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762863.479608:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d2680 x1876700024705664/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762888 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762863.479657:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762863.479665:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789762863.479674:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024705664:192.168.203.107@tcp:250:kworker.0 00000100:00100000:2.0:1789762863.479714:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024705792:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789762863.479772:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789762863.479792:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762863/real 1789762863] req@ffffa06c091d0380 x1876700024705792/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762888 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762863.479812:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0380 x1876700024705792/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762888 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762863.479829:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762863.479836:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789762863.479842:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024705792:192.168.203.107@tcp:38:kworker.0 00000100:00100000:2.0:1789762863.479915:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024705920:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789762863.479952:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789762863.479968:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762863/real 1789762863] req@ffffa06c091d3100 x1876700024705920/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762888 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762863.479984:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d3100 x1876700024705920/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762888 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762863.479994:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762863.479998:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789762863.480002:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024705920:192.168.203.107@tcp:38:kworker.0 00010000:00010000:0.0:1789762863.734943:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762863.990925:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762864.759024:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762865.014963:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762865.782969:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762866.038940:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789762866.807065:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762867.062912:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789762867.830981:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762868.086941:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789762868.598945:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762868.598956:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789762868.598960:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762868.598971:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2302 00000100:00080000:0.0:1789762868.598983:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762868.598992:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789762868.599034:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d1f80 x1876700024706048/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762868.599066:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762868.599069:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762868.599072:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762868.599076:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2302 00000100:00080000:0.0:1789762868.599079:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762868.599083:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789762868.599092:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762868.599103:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0a80 x1876700024706176/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762868.599115:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762868.599118:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762868.599120:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762868.599123:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2302 00000100:00080000:0.0:1789762868.599127:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762868.599131:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789762868.599136:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762868.599173:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0700 x1876700024706304/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789762868.599173:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024706048:192.168.203.107@tcp:250:kworker.0 00000800:00000100:2.0:1789762868.599275:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789762868.599319:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762868/real 1789762868] req@ffffa06c091d1f80 x1876700024706048/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762893 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762868.599338:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d1f80 x1876700024706048/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762893 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762868.599349:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762868.599353:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789762868.599357:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024706048:192.168.203.107@tcp:250:kworker.0 00000100:00100000:2.0:1789762868.599364:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024706176:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789762868.599383:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789762868.599394:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762868/real 1789762868] req@ffffa06c091d0a80 x1876700024706176/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762893 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762868.599406:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0a80 x1876700024706176/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762893 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762868.599414:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762868.599417:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789762868.599420:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024706176:192.168.203.107@tcp:38:kworker.0 00000100:00100000:2.0:1789762868.599443:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024706304:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789762868.599477:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789762868.599503:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762868/real 1789762868] req@ffffa06c091d0700 x1876700024706304/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762893 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762868.599515:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0700 x1876700024706304/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762893 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762868.599523:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762868.599527:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789762868.599530:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024706304:192.168.203.107@tcp:38:kworker.0 00010000:00010000:2.0:1789762868.854980:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789762869.110948:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789762869.878989:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789762870.134952:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762870.902950:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789762871.158967:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789762871.927001:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789762872.182976:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789762872.951006:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789762873.206941:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789762873.718975:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762873.718986:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789762873.718990:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762873.719002:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2307 00000100:00080000:0.0:1789762873.719014:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762873.719021:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789762873.719066:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0700 x1876700024706432/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762873.719093:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762873.719097:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762873.719098:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762873.719101:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2307 00000100:00080000:0.0:1789762873.719103:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762873.719105:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789762873.719115:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762873.719124:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0a80 x1876700024706560/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762873.719133:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762873.719154:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762873.719155:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762873.719159:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2307 00000100:00080000:0.0:1789762873.719164:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762873.719180:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789762873.719186:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762873.719200:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d1f80 x1876700024706688/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789762873.719218:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024706432:192.168.203.107@tcp:250:kworker.0 00000800:00000100:2.0:1789762873.719289:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789762873.719329:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762873/real 1789762873] req@ffffa06c091d0700 x1876700024706432/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762898 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762873.719343:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0700 x1876700024706432/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762898 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762873.719352:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762873.719355:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789762873.719358:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024706432:192.168.203.107@tcp:250:kworker.0 00000100:00100000:2.0:1789762873.719364:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024706560:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789762873.719378:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789762873.719385:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762873/real 1789762873] req@ffffa06c091d0a80 x1876700024706560/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762898 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762873.719411:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0a80 x1876700024706560/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762898 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762873.719417:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762873.719419:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789762873.719422:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024706560:192.168.203.107@tcp:38:kworker.0 00000100:00100000:2.0:1789762873.719440:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024706688:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789762873.719463:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789762873.719470:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762873/real 1789762873] req@ffffa06c091d1f80 x1876700024706688/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762898 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762873.719477:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d1f80 x1876700024706688/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762898 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762873.719482:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762873.719485:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789762873.719487:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024706688:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789762873.974941:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789762874.230929:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789762874.999049:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789762875.254942:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789762876.023024:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789762876.278928:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762877.046989:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789762877.302940:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789762878.070962:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789762878.326917:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:1.0:1789762878.838974:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762878.838982:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789762878.838985:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762878.838994:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2312 00000100:00080000:1.0:1789762878.839003:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762878.839008:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789762878.839039:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2d180 x1876700024706816/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762878.839062:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762878.839064:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762878.839066:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762878.839076:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2312 00000100:00080000:1.0:1789762878.839078:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762878.839081:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789762878.839088:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762878.839099:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2ca80 x1876700024706944/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762878.839107:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762878.839123:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762878.839124:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762878.839128:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2312 00000100:00080000:1.0:1789762878.839150:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1789762878.839157:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024706816:192.168.203.107@tcp:250:kworker.0 00000100:00080000:1.0:1789762878.839160:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789762878.839174:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762878.839194:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2c700 x1876700024707072/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789762878.839254:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789762878.839309:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762878/real 1789762878] req@ffffa06c08e2d180 x1876700024706816/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762903 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762878.839331:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2d180 x1876700024706816/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762903 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762878.839358:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762878.839361:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789762878.839365:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024706816:192.168.203.107@tcp:250:kworker.0 00000100:00100000:2.0:1789762878.839371:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024706944:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789762878.839392:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789762878.839401:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762878/real 1789762878] req@ffffa06c08e2ca80 x1876700024706944/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762903 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762878.839409:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2ca80 x1876700024706944/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762903 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762878.839415:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762878.839418:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789762878.839420:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024706944:192.168.203.107@tcp:38:kworker.0 00000100:00100000:2.0:1789762878.839445:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024707072:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789762878.839457:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789762878.839463:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762878/real 1789762878] req@ffffa06c08e2c700 x1876700024707072/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762903 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762878.839470:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2c700 x1876700024707072/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762903 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762878.839476:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762878.839478:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789762878.839480:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024707072:192.168.203.107@tcp:38:kworker.0 00010000:00010000:1.0:1789762879.095013:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789762879.350935:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762880.118966:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789762880.374911:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789762881.142967:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789762881.398912:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789762882.166997:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789762882.422920:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762883.190955:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789762883.447017:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:3.0:1789762883.958917:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762883.958930:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789762883.958935:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762883.958947:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2317 00000100:00080000:3.0:1789762883.958960:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762883.958969:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:3.0:1789762883.959014:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872ce00 x1876700024707200/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762883.959045:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762883.959062:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762883.959065:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762883.959070:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2317 00000100:00080000:3.0:1789762883.959076:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762883.959082:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789762883.959107:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762883.959128:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872d180 x1876700024707328/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789762883.959132:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024707200:192.168.203.107@tcp:250:kworker.0 00000100:00080000:3.0:1789762883.959162:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762883.959167:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762883.959169:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762883.959175:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2317 00000100:00080000:3.0:1789762883.959180:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762883.959185:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789762883.959193:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762883.959208:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872e300 x1876700024707456/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789762883.959260:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024707328:192.168.203.107@tcp:38:kworker.0 00000100:00100000:2.0:1789762883.959284:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024707456:192.168.203.107@tcp:38:kworker.0 00010000:00010000:2.0:1789762884.215025:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789762884.470949:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789762885.238956:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789762885.494917:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762886.263048:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789762886.519003:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000400:00000100:0.0:1789762887.031134:0:2329:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.107:988 00000400:00000100:0.0:1789762887.031201:0:2329:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.107@tcp at host 192.168.203.107 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1789762887.031218:0:2329:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.107@tcp 00000800:00000100:0.0:1789762887.031257:0:2329:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.107@tcp 00000800:00000100:0.0:1789762887.031267:0:2329:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.107@tcp 00000100:00100000:2.0:1789762887.031307:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762883/real 1789762887] req@ffffa06c1872d180 x1876700024707328/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762908 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762887.031334:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872d180 x1876700024707328/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762908 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762887.031347:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762887.031353:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789762887.031358:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024707328:192.168.203.107@tcp:38:kworker.0 00000100:00100000:2.0:1789762887.031366:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762883/real 1789762887] req@ffffa06c1872e300 x1876700024707456/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762908 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762887.031380:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872e300 x1876700024707456/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762908 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762887.031388:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762887.031393:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789762887.031397:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024707456:192.168.203.107@tcp:38:kworker.0 00000100:00100000:2.0:1789762887.031404:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762883/real 1789762887] req@ffffa06c1872ce00 x1876700024707200/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762908 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762887.031443:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872ce00 x1876700024707200/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762908 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762887.031451:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762887.031454:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789762887.031457:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024707200:192.168.203.107@tcp:250:kworker.0 00010000:00010000:3.0:1789762887.286991:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789762887.542912:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789762888.311048:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789762888.566929:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:3.0:1789762889.078912:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762889.078921:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789762889.078924:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762889.078934:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2322 00000100:00080000:3.0:1789762889.078942:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762889.078957:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:3.0:1789762889.079006:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872c380 x1876700024707584/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762889.079034:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762889.079039:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762889.079042:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762889.079047:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2322 00000100:00080000:3.0:1789762889.079052:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762889.079072:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789762889.079083:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762889.079101:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872d880 x1876700024707712/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762889.079117:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1789762889.079120:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024707584:192.168.203.107@tcp:250:kworker.0 00000100:00080000:3.0:1789762889.079123:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762889.079128:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762889.079136:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2322 00000100:00080000:3.0:1789762889.079141:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762889.079147:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789762889.079153:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762889.079168:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872df80 x1876700024707840/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789762889.079205:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762889.079244:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762889/real 1789762889] req@ffffa06c1872c380 x1876700024707584/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762914 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762889.079260:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872c380 x1876700024707584/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762914 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762889.079271:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762889.079276:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762889.079280:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024707584:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789762889.079316:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024707712:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789762889.079339:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762889.079352:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762889/real 1789762889] req@ffffa06c1872d880 x1876700024707712/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762914 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762889.079379:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872d880 x1876700024707712/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762914 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762889.079387:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762889.079391:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762889.079395:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024707712:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789762889.079401:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024707840:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789762889.079421:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762889.079433:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762889/real 1789762889] req@ffffa06c1872df80 x1876700024707840/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762914 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762889.079444:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872df80 x1876700024707840/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762914 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762889.079453:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762889.079456:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762889.079459:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024707840:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789762889.334965:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789762889.590918:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762890.358969:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789762890.614933:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789762891.382922:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789762891.638948:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762892.406959:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789762892.662947:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762893.430970:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789762893.686914:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:3.0:1789762894.198915:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762894.198922:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789762894.198925:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762894.198934:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2328 00000100:00080000:3.0:1789762894.198942:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762894.198948:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:3.0:1789762894.198988:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872f800 x1876700024707968/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762894.199011:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762894.199014:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762894.199015:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762894.199034:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2328 00000100:00080000:3.0:1789762894.199039:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762894.199044:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789762894.199053:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762894.199067:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024707968:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789762894.199073:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872df80 x1876700024708096/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762894.199089:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762894.199093:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762894.199095:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762894.199099:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2328 00000100:00080000:3.0:1789762894.199103:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762894.199107:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789762894.199112:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1789762894.199121:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789762894.199142:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872d880 x1876700024708224/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789762894.199156:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762894/real 1789762894] req@ffffa06c1872f800 x1876700024707968/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762919 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762894.199171:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872f800 x1876700024707968/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762919 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762894.199200:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762894.199204:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762894.199207:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024707968:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789762894.199227:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024708096:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789762894.199253:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762894.199264:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762894/real 1789762894] req@ffffa06c1872df80 x1876700024708096/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762919 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762894.199273:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872df80 x1876700024708096/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762919 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762894.199279:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762894.199281:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762894.199283:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024708096:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789762894.199288:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024708224:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789762894.199301:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762894.199307:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762894/real 1789762894] req@ffffa06c1872d880 x1876700024708224/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762919 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762894.199314:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872d880 x1876700024708224/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762919 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762894.199328:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762894.199339:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762894.199341:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024708224:192.168.203.107@tcp:38:kworker.0 00010000:00010000:0.0:1789762894.454972:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789762894.710957:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762895.478947:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789762895.734922:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762896.502982:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789762896.758942:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762897.527040:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789762897.782914:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762898.551013:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789762898.806952:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:3.0:1789762899.318959:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762899.318970:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789762899.318974:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762899.318996:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2333 00000100:00080000:3.0:1789762899.319009:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762899.319017:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:3.0:1789762899.319061:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872c380 x1876700024708352/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762899.319089:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762899.319092:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762899.319093:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762899.319097:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2333 00000100:00080000:3.0:1789762899.319099:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762899.319102:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789762899.319111:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762899.319123:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872ce00 x1876700024708480/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762899.319132:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762899.319135:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762899.319156:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1789762899.319158:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024708352:192.168.203.107@tcp:250:kworker.0 00000100:00080000:3.0:1789762899.319162:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2333 00000100:00080000:3.0:1789762899.319169:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762899.319173:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789762899.319177:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762899.319186:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872e300 x1876700024708608/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789762899.319247:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762899.319297:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762899/real 1789762899] req@ffffa06c1872c380 x1876700024708352/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762924 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762899.319317:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872c380 x1876700024708352/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762924 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762899.319328:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762899.319333:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762899.319337:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024708352:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789762899.319345:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024708480:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789762899.319365:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762899.319377:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762899/real 1789762899] req@ffffa06c1872ce00 x1876700024708480/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762924 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762899.319390:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872ce00 x1876700024708480/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762924 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762899.319400:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762899.319404:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762899.319408:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024708480:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789762899.319434:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024708608:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789762899.319450:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762899.319457:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762899/real 1789762899] req@ffffa06c1872e300 x1876700024708608/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762924 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762899.319473:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872e300 x1876700024708608/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762924 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762899.319478:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762899.319480:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762899.319483:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024708608:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789762899.574963:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789762899.830918:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789762900.599020:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789762900.855085:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762901.623016:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789762901.878941:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762902.646988:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789762902.902914:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762903.670984:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789762903.926979:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:3.0:1789762904.438904:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762904.438913:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789762904.438916:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762904.438924:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2338 00000100:00080000:3.0:1789762904.438934:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762904.438940:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:3.0:1789762904.438985:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872d180 x1876700024708736/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762904.439008:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762904.439010:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762904.439012:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762904.439015:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2338 00000100:00080000:3.0:1789762904.439018:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762904.439020:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789762904.439038:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762904.439050:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872ca80 x1876700024708864/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762904.439060:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762904.439063:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762904.439065:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1789762904.439070:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024708736:192.168.203.107@tcp:250:kworker.0 00000100:00080000:3.0:1789762904.439070:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2338 00000100:00080000:3.0:1789762904.439076:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762904.439080:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789762904.439083:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762904.439093:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872f100 x1876700024708992/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789762904.439128:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762904.439162:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762904/real 1789762904] req@ffffa06c1872d180 x1876700024708736/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762929 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762904.439175:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872d180 x1876700024708736/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762929 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762904.439183:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762904.439185:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762904.439189:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024708736:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789762904.439206:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024708864:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789762904.439220:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762904.439228:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762904/real 1789762904] req@ffffa06c1872ca80 x1876700024708864/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762929 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762904.439235:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872ca80 x1876700024708864/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762929 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762904.439251:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762904.439253:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762904.439256:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024708864:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789762904.439260:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024708992:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789762904.439273:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762904.439279:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762904/real 1789762904] req@ffffa06c1872f100 x1876700024708992/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762929 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762904.439287:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872f100 x1876700024708992/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762929 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762904.439292:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762904.439293:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762904.439295:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024708992:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789762904.694976:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789762904.950936:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762905.718964:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789762905.974969:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789762906.742973:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789762906.998922:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762907.766984:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789762908.022941:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789762908.790998:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789762909.046923:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:3.0:1789762909.558945:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762909.558965:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789762909.558968:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762909.558976:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2343 00000100:00080000:3.0:1789762909.558984:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762909.558990:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:3.0:1789762909.559019:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872c700 x1876700024709120/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762909.559042:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762909.559045:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762909.559058:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762909.559061:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2343 00000100:00080000:3.0:1789762909.559065:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762909.559068:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789762909.559076:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762909.559085:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872e680 x1876700024709248/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762909.559095:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1789762909.559099:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024709120:192.168.203.107@tcp:250:kworker.0 00000100:00080000:3.0:1789762909.559099:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762909.559101:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762909.559109:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2343 00000100:00080000:3.0:1789762909.559112:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762909.559114:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789762909.559117:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762909.559126:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872dc00 x1876700024709376/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789762909.559166:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762909.559206:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762909/real 1789762909] req@ffffa06c1872c700 x1876700024709120/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762934 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762909.559224:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872c700 x1876700024709120/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762934 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762909.559235:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762909.559240:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762909.559251:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024709120:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789762909.559274:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024709248:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789762909.559298:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762909.559312:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762909/real 1789762909] req@ffffa06c1872e680 x1876700024709248/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762934 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762909.559325:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872e680 x1876700024709248/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762934 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762909.559334:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762909.559338:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762909.559341:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024709248:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789762909.559348:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024709376:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789762909.559368:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762909.559379:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762909/real 1789762909] req@ffffa06c1872dc00 x1876700024709376/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762934 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762909.559402:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872dc00 x1876700024709376/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762934 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762909.559412:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762909.559415:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762909.559419:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024709376:192.168.203.107@tcp:38:kworker.0 00010000:00010000:1.0:1789762909.814999:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789762910.070923:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762910.839059:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789762911.094992:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762911.863001:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789762912.118987:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789762912.886983:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789762913.142982:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762913.911060:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789762914.167003:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:3.0:1789762914.679032:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762914.679050:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789762914.679057:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762914.679075:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2348 00000100:00080000:3.0:1789762914.679095:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762914.679107:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:3.0:1789762914.679172:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872ed80 x1876700024709504/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762914.679252:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762914.679259:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762914.679263:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762914.679272:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2348 00000100:00080000:3.0:1789762914.679281:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762914.679292:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789762914.679311:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762914.679342:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024709504:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789762914.679346:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872ea00 x1876700024709632/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762914.679379:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762914.679387:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762914.679391:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762914.679400:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2348 00000100:00080000:3.0:1789762914.679408:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762914.679417:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789762914.679427:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1789762914.679446:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789762914.679455:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872f480 x1876700024709760/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789762914.679516:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762914/real 1789762914] req@ffffa06c1872ed80 x1876700024709504/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762939 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762914.679555:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872ed80 x1876700024709504/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762939 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762914.679576:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762914.679584:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762914.679592:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024709504:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789762914.679634:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024709632:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789762914.679721:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762914.679750:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762914/real 1789762914] req@ffffa06c1872ea00 x1876700024709632/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762939 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762914.679777:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872ea00 x1876700024709632/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762939 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762914.679794:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762914.679801:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762914.679808:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024709632:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789762914.679822:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024709760:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789762914.679967:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762914.679995:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762914/real 1789762914] req@ffffa06c1872f480 x1876700024709760/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762939 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762914.680044:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872f480 x1876700024709760/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762939 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762914.680054:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762914.680059:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762914.680063:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024709760:192.168.203.107@tcp:38:kworker.0 00010000:00010000:2.0:1789762914.935076:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789762915.190930:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789762915.958981:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789762916.214960:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762916.982996:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789762917.239004:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762918.007157:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789762918.262942:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789762919.030974:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789762919.286941:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:1.0:1789762919.798966:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762919.798973:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789762919.798976:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762919.798985:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2353 00000100:00080000:1.0:1789762919.798994:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762919.799000:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789762919.799031:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2c000 x1876700024709888/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762919.799054:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762919.799057:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762919.799058:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762919.799061:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2353 00000100:00080000:1.0:1789762919.799063:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762919.799065:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789762919.799072:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762919.799081:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2e300 x1876700024710016/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762919.799103:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762919.799107:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762919.799110:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1789762919.799110:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024709888:192.168.203.107@tcp:250:kworker.0 00000100:00080000:1.0:1789762919.799118:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2353 00000100:00080000:1.0:1789762919.799122:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762919.799124:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789762919.799128:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762919.799137:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2ed80 x1876700024710144/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789762919.799173:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789762919.799205:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762919/real 1789762919] req@ffffa06c08e2c000 x1876700024709888/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762944 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762919.799218:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2c000 x1876700024709888/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762944 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762919.799226:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762919.799228:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789762919.799231:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024709888:192.168.203.107@tcp:250:kworker.0 00000100:00100000:2.0:1789762919.799252:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024710016:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789762919.799283:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789762919.799296:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762919/real 1789762919] req@ffffa06c08e2e300 x1876700024710016/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762944 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762919.799318:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2e300 x1876700024710016/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762944 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762919.799324:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762919.799326:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789762919.799329:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024710016:192.168.203.107@tcp:38:kworker.0 00000100:00100000:2.0:1789762919.799333:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024710144:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789762919.799348:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789762919.799354:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762919/real 1789762919] req@ffffa06c08e2ed80 x1876700024710144/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762944 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762919.799362:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2ed80 x1876700024710144/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762944 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762919.799367:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762919.799368:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789762919.799370:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024710144:192.168.203.107@tcp:38:kworker.0 00010000:00010000:2.0:1789762920.054950:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789762920.310926:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762921.079045:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789762921.334936:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762922.102968:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789762922.358926:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762923.126969:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789762923.382906:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762924.150952:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789762924.406921:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789762924.919000:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762924.919008:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789762924.919012:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762924.919023:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2358 00000100:00080000:0.0:1789762924.919034:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762924.919041:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789762924.919074:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d3100 x1876700024710272/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762924.919097:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762924.919100:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762924.919101:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762924.919106:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2358 00000100:00080000:0.0:1789762924.919109:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762924.919112:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789762924.919122:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762924.919135:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0380 x1876700024710400/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789762924.919144:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024710272:192.168.203.107@tcp:250:kworker.0 00000100:00080000:0.0:1789762924.919175:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762924.919178:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762924.919181:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762924.919186:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2358 00000100:00080000:0.0:1789762924.919191:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762924.919197:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789762924.919201:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762924.919214:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d2680 x1876700024710528/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789762924.919239:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762924.919283:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762924/real 1789762924] req@ffffa06c091d3100 x1876700024710272/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762949 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762924.919298:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d3100 x1876700024710272/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762949 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762924.919307:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762924.919310:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762924.919314:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024710272:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789762924.919338:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024710400:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789762924.919354:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762924.919363:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762924/real 1789762924] req@ffffa06c091d0380 x1876700024710400/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762949 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762924.919385:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0380 x1876700024710400/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762949 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762924.919392:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762924.919395:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762924.919397:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024710400:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789762924.919401:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024710528:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789762924.919415:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762924.919422:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762924/real 1789762924] req@ffffa06c091d2680 x1876700024710528/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762949 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762924.919430:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d2680 x1876700024710528/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762949 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762924.919435:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762924.919437:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762924.919439:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024710528:192.168.203.107@tcp:38:kworker.0 00010000:00010000:2.0:1789762925.174969:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789762925.430957:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789762926.198967:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789762926.454915:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762927.222969:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789762927.478944:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762928.246970:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789762928.502914:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789762929.271117:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789762929.526944:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:1.0:1789762930.038978:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762930.038988:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789762930.038992:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762930.039002:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2363 00000100:00080000:1.0:1789762930.039012:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762930.039019:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789762930.039074:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2d500 x1876700024710656/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762930.039108:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762930.039112:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762930.039114:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762930.039119:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2363 00000100:00080000:1.0:1789762930.039123:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762930.039127:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789762930.039147:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762930.039172:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2ed80 x1876700024710784/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762930.039186:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1789762930.039189:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024710656:192.168.203.107@tcp:250:kworker.0 00000100:00080000:1.0:1789762930.039191:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762930.039199:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762930.039206:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2363 00000100:00080000:1.0:1789762930.039209:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762930.039214:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789762930.039220:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762930.039234:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2e300 x1876700024710912/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789762930.039253:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789762930.039288:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762930/real 1789762930] req@ffffa06c08e2d500 x1876700024710656/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762955 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762930.039302:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2d500 x1876700024710656/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762955 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762930.039310:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762930.039313:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789762930.039315:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024710656:192.168.203.107@tcp:250:kworker.0 00000100:00100000:2.0:1789762930.039381:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024710784:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789762930.039398:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789762930.039407:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762930/real 1789762930] req@ffffa06c08e2ed80 x1876700024710784/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762955 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762930.039415:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2ed80 x1876700024710784/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762955 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762930.039420:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762930.039423:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789762930.039425:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024710784:192.168.203.107@tcp:38:kworker.0 00000100:00100000:2.0:1789762930.039429:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024710912:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789762930.039439:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789762930.039445:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762930/real 1789762930] req@ffffa06c08e2e300 x1876700024710912/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762955 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762930.039460:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2e300 x1876700024710912/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762955 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762930.039465:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762930.039467:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789762930.039469:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024710912:192.168.203.107@tcp:38:kworker.0 00010000:00010000:2.0:1789762930.294965:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789762930.551022:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762931.319038:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762931.575001:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789762932.343020:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762932.598995:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762933.367175:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762933.622992:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762934.391040:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762934.646915:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789762935.159045:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762935.159058:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789762935.159063:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762935.159075:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2368 00000100:00080000:2.0:1789762935.159088:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762935.159096:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789762935.159188:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51880 x1876700024711040/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762935.159221:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762935.159225:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762935.159226:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762935.159230:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2368 00000100:00080000:2.0:1789762935.159233:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762935.159235:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789762935.159268:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762935.159280:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e52300 x1876700024711168/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762935.159291:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762935.159294:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762935.159296:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762935.159299:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2368 00000100:00080000:2.0:1789762935.159303:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762935.159306:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789762935.159311:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762935.159319:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51180 x1876700024711296/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789762935.159330:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024711040:192.168.203.107@tcp:250:kworker.0 00000800:00000100:1.0:1789762935.159480:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762935.159547:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762935/real 1789762935] req@ffffa06c07e51880 x1876700024711040/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762960 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762935.159594:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51880 x1876700024711040/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762960 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762935.159610:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762935.159616:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762935.159621:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024711040:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789762935.159633:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e52300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024711168:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789762935.159667:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762935.159684:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762935/real 1789762935] req@ffffa06c07e52300 x1876700024711168/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762960 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762935.159699:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e52300 x1876700024711168/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762960 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762935.159710:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762935.159714:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762935.159741:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e52300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024711168:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789762935.159774:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024711296:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789762935.159809:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762935.159831:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762935/real 1789762935] req@ffffa06c07e51180 x1876700024711296/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762960 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762935.159899:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51180 x1876700024711296/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762960 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762935.159928:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762935.159933:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762935.159937:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024711296:192.168.203.107@tcp:38:kworker.0 00010000:00010000:0.0:1789762935.415031:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762935.670935:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762936.439041:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762936.694928:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789762937.463078:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762937.719026:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762938.487010:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762938.742911:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789762939.511026:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762939.766917:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789762940.278977:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762940.278988:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789762940.278993:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762940.279004:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2374 00000100:00080000:2.0:1789762940.279016:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762940.279024:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789762940.279078:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50000 x1876700024711424/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762940.279103:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762940.279106:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762940.279125:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762940.279130:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2374 00000100:00080000:2.0:1789762940.279135:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762940.279140:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789762940.279152:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762940.279169:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51500 x1876700024711552/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762940.279184:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762940.279187:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762940.279189:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762940.279193:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2374 00000100:00080000:2.0:1789762940.279198:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762940.279202:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789762940.279206:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762940.279217:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024711424:192.168.203.107@tcp:250:kworker.0 00000100:00100000:2.0:1789762940.279218:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50e00 x1876700024711680/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789762940.279297:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762940.279334:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762940/real 1789762940] req@ffffa06c07e50000 x1876700024711424/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762965 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762940.279348:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50000 x1876700024711424/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762965 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762940.279356:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762940.279359:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762940.279362:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024711424:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789762940.279367:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024711552:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789762940.279381:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762940.279389:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762940/real 1789762940] req@ffffa06c07e51500 x1876700024711552/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762965 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762940.279396:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51500 x1876700024711552/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762965 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762940.279402:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762940.279404:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762940.279419:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024711552:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789762940.279440:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024711680:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789762940.279454:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762940.279461:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762940/real 1789762940] req@ffffa06c07e50e00 x1876700024711680/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762965 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762940.279469:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50e00 x1876700024711680/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762965 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762940.279474:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762940.279476:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762940.279478:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024711680:192.168.203.107@tcp:38:kworker.0 00010000:00010000:1.0:1789762940.534984:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762940.790909:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762941.558995:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762941.814959:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789762942.582997:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762942.838959:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789762943.607003:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762943.862920:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762944.630974:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762944.886929:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789762945.398982:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762945.398993:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789762945.398997:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762945.399007:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2379 00000100:00080000:0.0:1789762945.399018:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762945.399025:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789762945.399062:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d2a00 x1876700024711808/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762945.399104:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762945.399109:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762945.399111:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762945.399116:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2379 00000100:00080000:0.0:1789762945.399121:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762945.399127:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789762945.399138:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762945.399145:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024711808:192.168.203.107@tcp:250:kworker.0 00000100:00100000:0.0:1789762945.399155:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0e00 x1876700024711936/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762945.399169:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762945.399174:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762945.399176:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762945.399180:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2379 00000100:00080000:0.0:1789762945.399193:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1789762945.399199:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00080000:0.0:1789762945.399208:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789762945.399215:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762945.399230:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d1500 x1876700024712064/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789762945.399242:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762945/real 1789762945] req@ffffa06c091d2a00 x1876700024711808/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762970 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762945.399265:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d2a00 x1876700024711808/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762970 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762945.399275:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762945.399277:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762945.399280:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024711808:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789762945.399321:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024711936:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789762945.399339:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762945.399348:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762945/real 1789762945] req@ffffa06c091d0e00 x1876700024711936/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762970 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762945.399356:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0e00 x1876700024711936/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762970 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762945.399362:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762945.399364:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762945.399366:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024711936:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789762945.399371:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024712064:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789762945.399389:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762945.399400:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762945/real 1789762945] req@ffffa06c091d1500 x1876700024712064/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762970 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762945.399413:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d1500 x1876700024712064/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762970 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762945.399422:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762945.399425:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762945.399428:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024712064:192.168.203.107@tcp:38:kworker.0 00010000:00010000:1.0:1789762945.654989:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762945.910918:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762946.679071:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762946.935039:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762947.703052:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762947.958978:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789762948.727077:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762948.982939:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762949.751033:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762950.006939:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:1.0:1789762950.519000:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762950.519053:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789762950.519059:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762950.519073:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2384 00000100:00080000:1.0:1789762950.519090:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762950.519101:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789762950.519154:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2c000 x1876700024712192/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762950.519202:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762950.519207:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762950.519210:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762950.519216:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2384 00000100:00080000:1.0:1789762950.519220:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762950.519225:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789762950.519268:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762950.519295:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2c700 x1876700024712320/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789762950.519316:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024712192:192.168.203.107@tcp:250:kworker.0 00000100:00080000:1.0:1789762950.519319:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762950.519335:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762950.519339:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762950.519347:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2384 00000100:00080000:1.0:1789762950.519354:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762950.519363:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789762950.519374:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762950.519398:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2ca80 x1876700024712448/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789762950.519485:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024712320:192.168.203.107@tcp:38:kworker.0 00000100:00100000:2.0:1789762950.519539:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024712448:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789762950.774998:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762951.030903:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762951.799007:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762952.054999:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762952.822962:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762953.078952:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000400:00000100:3.0:1789762953.591226:0:2330:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.107:988 00000400:00000100:3.0:1789762953.592210:0:2330:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.107@tcp at host 192.168.203.107 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1789762953.592230:0:2330:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.107@tcp 00000800:00000100:3.0:1789762953.592284:0:2330:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.107@tcp 00000800:00000100:3.0:1789762953.592313:0:2330:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.107@tcp 00000100:00100000:1.0:1789762953.592367:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762950/real 1789762953] req@ffffa06c08e2c700 x1876700024712320/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762975 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762953.592416:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2c700 x1876700024712320/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762975 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762953.592430:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762953.592436:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762953.592441:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024712320:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789762953.592469:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762950/real 1789762953] req@ffffa06c08e2ca80 x1876700024712448/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762975 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762953.592488:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2ca80 x1876700024712448/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762975 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762953.592501:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762953.592507:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762953.592513:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024712448:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789762953.592523:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762950/real 1789762953] req@ffffa06c08e2c000 x1876700024712192/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762975 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762953.592542:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2c000 x1876700024712192/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762975 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762953.592555:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762953.592560:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762953.592608:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024712192:192.168.203.107@tcp:250:kworker.0 00010000:00010000:3.0:1789762953.846979:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762954.102927:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762954.871056:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762955.126978:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789762955.638990:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762955.639000:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789762955.639004:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762955.639012:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2389 00000100:00080000:2.0:1789762955.639021:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762955.639026:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789762955.639060:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51f80 x1876700024712576/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762955.639078:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762955.639082:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762955.639084:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762955.639089:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2389 00000100:00080000:2.0:1789762955.639092:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762955.639096:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789762955.639105:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762955.639115:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024712576:192.168.203.107@tcp:250:kworker.0 00000100:00100000:2.0:1789762955.639118:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e53b80 x1876700024712704/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762955.639128:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762955.639132:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762955.639133:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762955.639136:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2389 00000100:00080000:2.0:1789762955.639140:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762955.639143:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789762955.639163:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762955.639185:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50380 x1876700024712832/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789762955.639186:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762955.639223:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762955/real 1789762955] req@ffffa06c07e51f80 x1876700024712576/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762980 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762955.639243:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51f80 x1876700024712576/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762980 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762955.639254:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762955.639258:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762955.639262:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024712576:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789762955.639286:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e53b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024712704:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789762955.639311:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762955.639324:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762955/real 1789762955] req@ffffa06c07e53b80 x1876700024712704/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762980 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762955.639338:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e53b80 x1876700024712704/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762980 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762955.639347:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762955.639351:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762955.639354:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e53b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024712704:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789762955.639360:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024712832:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789762955.639376:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762955.639398:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762955/real 1789762955] req@ffffa06c07e50380 x1876700024712832/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762980 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762955.639410:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50380 x1876700024712832/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762980 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762955.639419:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762955.639422:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762955.639425:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024712832:192.168.203.107@tcp:38:kworker.0 00010000:00010000:2.0:1789762955.895099:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762956.150983:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789762956.918974:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762957.174919:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789762957.943069:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762958.198914:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762958.967013:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762959.222925:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789762959.991041:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789762960.246930:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789762960.758998:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762960.759005:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789762960.759008:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762960.759017:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2394 00000100:00080000:2.0:1789762960.759026:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762960.759033:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789762960.759070:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e52680 x1876700024712960/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762960.759099:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762960.759102:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762960.759104:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762960.759107:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2394 00000100:00080000:2.0:1789762960.759110:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762960.759113:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789762960.759121:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762960.759136:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e53800 x1876700024713088/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762960.759146:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762960.759151:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:1.0:1789762960.759164:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e52680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024712960:192.168.203.107@tcp:250:kworker.0 00000100:00080000:2.0:1789762960.759170:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762960.759178:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2394 00000100:00080000:2.0:1789762960.759183:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762960.759189:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789762960.759197:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762960.759213:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51c00 x1876700024713216/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789762960.759234:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762960.759277:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762960/real 1789762960] req@ffffa06c07e52680 x1876700024712960/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762985 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762960.759296:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e52680 x1876700024712960/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762985 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762960.759307:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762960.759313:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762960.759317:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e52680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024712960:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789762960.759325:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e53800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024713088:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789762960.759352:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762960.759373:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762960/real 1789762960] req@ffffa06c07e53800 x1876700024713088/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762985 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762960.759403:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e53800 x1876700024713088/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762985 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762960.759413:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762960.759417:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762960.759421:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e53800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024713088:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789762960.759449:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024713216:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789762960.759467:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762960.759475:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762960/real 1789762960] req@ffffa06c07e51c00 x1876700024713216/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762985 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762960.759484:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51c00 x1876700024713216/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762985 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762960.759489:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762960.759492:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762960.759494:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024713216:192.168.203.107@tcp:38:kworker.0 00010000:00010000:1.0:1789762961.014943:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789762961.270941:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762962.038944:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762962.294969:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789762963.062946:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762963.318928:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789762964.086978:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762964.342955:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762965.110957:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762965.366914:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789762965.878992:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762965.879003:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789762965.879024:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762965.879036:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2399 00000100:00080000:0.0:1789762965.879047:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762965.879055:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789762965.879087:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0000 x1876700024713344/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762965.879112:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762965.879116:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762965.879119:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762965.879124:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2399 00000100:00080000:0.0:1789762965.879128:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762965.879133:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789762965.879140:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024713344:192.168.203.107@tcp:250:kworker.0 00000008:00100000:0.0:1789762965.879143:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762965.879160:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d3b80 x1876700024713472/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762965.879174:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762965.879178:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762965.879180:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762965.879184:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2399 00000100:00080000:0.0:1789762965.879199:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762965.879204:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000800:00000100:1.0:1789762965.879209:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000008:00100000:0.0:1789762965.879212:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762965.879229:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d2300 x1876700024713600/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789762965.879245:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762965/real 1789762965] req@ffffa06c091d0000 x1876700024713344/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762990 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762965.879288:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0000 x1876700024713344/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762990 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762965.879297:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762965.879300:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762965.879303:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024713344:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789762965.879322:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024713472:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789762965.879346:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762965.879359:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762965/real 1789762965] req@ffffa06c091d3b80 x1876700024713472/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762990 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762965.879371:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d3b80 x1876700024713472/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762990 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762965.879379:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762965.879383:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762965.879387:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024713472:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789762965.879393:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024713600:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789762965.879412:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789762965.879423:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762965/real 1789762965] req@ffffa06c091d2300 x1876700024713600/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762990 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762965.879433:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d2300 x1876700024713600/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762990 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762965.879452:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762965.879455:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789762965.879458:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024713600:192.168.203.107@tcp:38:kworker.0 00010000:00010000:1.0:1789762966.134962:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762966.390899:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789762967.158962:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762967.414893:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762968.182976:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762968.438953:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789762969.207010:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762969.462963:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762970.230999:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762970.486941:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:1.0:1789762970.998977:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762970.998988:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789762970.998992:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762970.999003:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2404 00000100:00080000:1.0:1789762970.999015:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762970.999022:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789762970.999064:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2c000 x1876700024713728/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762970.999094:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762970.999098:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762970.999099:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762970.999103:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2404 00000100:00080000:1.0:1789762970.999105:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762970.999108:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789762970.999117:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762970.999127:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024713728:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789762970.999128:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2ca80 x1876700024713856/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762970.999161:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762970.999164:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762970.999165:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762970.999169:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2404 00000100:00080000:1.0:1789762970.999172:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762970.999175:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789762970.999179:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762970.999195:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2c700 x1876700024713984/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789762970.999206:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789762970.999257:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762970/real 1789762970] req@ffffa06c08e2c000 x1876700024713728/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762995 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762970.999272:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2c000 x1876700024713728/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762995 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762970.999281:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762970.999285:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789762970.999288:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024713728:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789762970.999313:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024713856:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789762970.999328:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789762970.999337:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762970/real 1789762970] req@ffffa06c08e2ca80 x1876700024713856/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762995 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762970.999357:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2ca80 x1876700024713856/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762995 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762970.999364:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762970.999366:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789762970.999368:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024713856:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789762970.999372:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024713984:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789762970.999386:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789762970.999393:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762970/real 1789762970] req@ffffa06c08e2c700 x1876700024713984/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762995 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762970.999401:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2c700 x1876700024713984/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762995 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762970.999406:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762970.999408:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789762970.999410:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024713984:192.168.203.107@tcp:38:kworker.0 00010000:00010000:1.0:1789762971.254970:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762971.510917:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789762972.278995:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762972.535098:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762973.302957:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762973.558945:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762974.326973:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762974.582947:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789762975.351037:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762975.607024:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789762976.119013:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762976.119025:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789762976.119030:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762976.119043:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2409 00000100:00080000:2.0:1789762976.119057:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762976.119066:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789762976.119118:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e53480 x1876700024714112/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762976.119154:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762976.119158:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762976.119160:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762976.119164:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2409 00000100:00080000:2.0:1789762976.119168:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762976.119171:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789762976.119219:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762976.119233:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50a80 x1876700024714240/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762976.119265:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762976.119272:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:3.0:1789762976.119275:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e53480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024714112:192.168.203.107@tcp:250:kworker.0 00000100:00080000:2.0:1789762976.119276:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762976.119284:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2409 00000100:00080000:2.0:1789762976.119294:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762976.119298:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789762976.119304:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762976.119315:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50700 x1876700024714368/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789762976.119404:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789762976.119520:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762976/real 1789762976] req@ffffa06c07e53480 x1876700024714112/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763001 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762976.119551:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e53480 x1876700024714112/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763001 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762976.119569:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762976.119576:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789762976.119582:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e53480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024714112:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789762976.119619:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024714240:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789762976.119658:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789762976.119678:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762976/real 1789762976] req@ffffa06c07e50a80 x1876700024714240/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763001 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762976.119699:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50a80 x1876700024714240/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763001 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762976.119713:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762976.119717:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789762976.119723:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024714240:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789762976.119734:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024714368:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789762976.119782:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789762976.119798:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762976/real 1789762976] req@ffffa06c07e50700 x1876700024714368/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763001 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762976.119815:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50700 x1876700024714368/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763001 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762976.119861:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762976.119867:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789762976.119873:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024714368:192.168.203.107@tcp:38:kworker.0 00010000:00010000:1.0:1789762976.374978:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762976.630952:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789762977.399010:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762977.654904:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789762978.422967:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762978.678922:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789762979.447083:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762979.702971:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789762980.471017:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762980.726959:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789762981.238962:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762981.238974:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789762981.238979:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762981.238992:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2415 00000100:00080000:2.0:1789762981.239006:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762981.239016:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789762981.239077:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e52d80 x1876700024714496/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762981.239115:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762981.239121:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762981.239123:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762981.239128:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2415 00000100:00080000:2.0:1789762981.239156:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762981.239163:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789762981.239176:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762981.239196:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e53100 x1876700024714624/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762981.239215:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762981.239223:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:3.0:1789762981.239224:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e52d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024714496:192.168.203.107@tcp:250:kworker.0 00000100:00080000:2.0:1789762981.239226:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762981.239239:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2415 00000100:00080000:2.0:1789762981.239248:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762981.239253:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789762981.239259:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762981.239277:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e52a00 x1876700024714752/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789762981.239332:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789762981.239430:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762981/real 1789762981] req@ffffa06c07e52d80 x1876700024714496/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763006 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762981.239456:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e52d80 x1876700024714496/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763006 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762981.239470:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762981.239476:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789762981.239482:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e52d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024714496:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789762981.239512:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e53100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024714624:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789762981.239543:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789762981.239559:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762981/real 1789762981] req@ffffa06c07e53100 x1876700024714624/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763006 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762981.239575:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e53100 x1876700024714624/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763006 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762981.239598:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762981.239603:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789762981.239608:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e53100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024714624:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789762981.239615:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e52a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024714752:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789762981.239642:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789762981.239657:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762981/real 1789762981] req@ffffa06c07e52a00 x1876700024714752/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763006 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762981.239671:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e52a00 x1876700024714752/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763006 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762981.239681:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762981.239685:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789762981.239688:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e52a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024714752:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789762981.495029:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762981.750980:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762982.519069:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762982.775049:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789762983.543107:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762983.798969:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762984.566997:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762984.822955:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762985.590959:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762985.846930:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:1.0:1789762986.358970:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762986.358977:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789762986.358981:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762986.358988:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2420 00000100:00080000:1.0:1789762986.358997:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762986.359003:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789762986.359036:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2d180 x1876700024714880/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762986.359084:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762986.359086:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762986.359088:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762986.359091:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2420 00000100:00080000:1.0:1789762986.359094:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762986.359097:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789762986.359105:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762986.359117:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2c700 x1876700024715008/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762986.359126:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762986.359128:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762986.359131:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1789762986.359133:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024714880:192.168.203.107@tcp:250:kworker.0 00000100:00080000:1.0:1789762986.359136:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2420 00000100:00080000:1.0:1789762986.359143:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762986.359145:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789762986.359149:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762986.359157:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2ca80 x1876700024715136/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789762986.359212:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789762986.359251:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762986/real 1789762986] req@ffffa06c08e2d180 x1876700024714880/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763011 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762986.359278:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2d180 x1876700024714880/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763011 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762986.359302:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762986.359305:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789762986.359308:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024714880:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789762986.359325:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024715008:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789762986.359360:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789762986.359371:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762986/real 1789762986] req@ffffa06c08e2c700 x1876700024715008/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763011 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762986.359380:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2c700 x1876700024715008/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763011 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762986.359385:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762986.359387:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789762986.359390:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024715008:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789762986.359393:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024715136:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789762986.359407:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789762986.359413:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762986/real 1789762986] req@ffffa06c08e2ca80 x1876700024715136/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763011 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762986.359421:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2ca80 x1876700024715136/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763011 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762986.359435:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762986.359437:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789762986.359439:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024715136:192.168.203.107@tcp:38:kworker.0 00010000:00010000:2.0:1789762986.614987:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762986.870934:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762987.639033:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762987.894943:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762988.662975:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762988.918938:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762989.686953:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762989.942981:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762990.711074:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762990.967038:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:1.0:1789762991.478950:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762991.478968:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789762991.478976:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762991.478996:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2425 00000100:00080000:1.0:1789762991.479049:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762991.479061:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789762991.479129:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2c000 x1876700024715264/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762991.479185:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762991.479194:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762991.479198:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762991.479206:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2425 00000100:00080000:1.0:1789762991.479213:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762991.479221:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789762991.479237:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762991.479260:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024715264:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789762991.479264:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2e300 x1876700024715392/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762991.479290:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762991.479295:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762991.479297:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762991.479330:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2425 00000100:00080000:1.0:1789762991.479337:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762991.479344:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789762991.479353:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762991.479375:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2ed80 x1876700024715520/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789762991.479425:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789762991.479507:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762991/real 1789762991] req@ffffa06c08e2c000 x1876700024715264/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763016 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762991.479536:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2c000 x1876700024715264/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763016 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762991.479552:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762991.479558:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789762991.479563:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024715264:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789762991.479601:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024715392:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789762991.479638:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789762991.479663:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762991/real 1789762991] req@ffffa06c08e2e300 x1876700024715392/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763016 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762991.479721:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2e300 x1876700024715392/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763016 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762991.479740:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762991.479745:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789762991.479750:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024715392:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789762991.479759:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024715520:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789762991.479792:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789762991.479808:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762991/real 1789762991] req@ffffa06c08e2ed80 x1876700024715520/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763016 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762991.479824:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2ed80 x1876700024715520/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763016 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762991.479834:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762991.479838:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789762991.479842:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024715520:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789762991.735043:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762991.990975:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762992.759054:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762993.014963:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789762993.783051:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762994.039008:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762994.807028:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762995.062902:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762995.830959:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762996.086900:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:1.0:1789762996.598934:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762996.598958:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789762996.598963:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762996.598973:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2430 00000100:00080000:1.0:1789762996.598982:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762996.598988:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789762996.599019:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2d500 x1876700024715648/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762996.599047:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762996.599050:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762996.599052:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762996.599055:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2430 00000100:00080000:1.0:1789762996.599059:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762996.599062:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789762996.599070:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762996.599081:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2ed80 x1876700024715776/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762996.599090:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762996.599093:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762996.599094:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762996.599097:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2430 00000100:00080000:1.0:1789762996.599100:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762996.599103:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789762996.599107:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762996.599115:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2e300 x1876700024715904/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789762996.599127:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024715648:192.168.203.107@tcp:250:kworker.0 00000800:00000100:2.0:1789762996.599182:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789762996.599212:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762996/real 1789762996] req@ffffa06c08e2d500 x1876700024715648/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763021 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762996.599239:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2d500 x1876700024715648/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763021 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762996.599246:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762996.599249:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789762996.599252:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024715648:192.168.203.107@tcp:250:kworker.0 00000100:00100000:2.0:1789762996.599258:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024715776:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789762996.599291:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789762996.599300:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762996/real 1789762996] req@ffffa06c08e2ed80 x1876700024715776/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763021 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762996.599308:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2ed80 x1876700024715776/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763021 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762996.599314:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762996.599316:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789762996.599318:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024715776:192.168.203.107@tcp:38:kworker.0 00000100:00100000:2.0:1789762996.599336:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024715904:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789762996.599347:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789762996.599353:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762996/real 1789762996] req@ffffa06c08e2e300 x1876700024715904/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763021 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762996.599360:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2e300 x1876700024715904/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763021 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762996.599375:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762996.599378:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789762996.599380:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024715904:192.168.203.107@tcp:38:kworker.0 00010000:00010000:2.0:1789762996.855104:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789762997.110950:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762997.878982:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789762998.134924:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789762998.903039:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789762999.158909:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789762999.926960:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789763000.182911:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763000.950939:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789763001.206930:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:1.0:1789763001.718909:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763001.718917:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789763001.718920:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763001.718928:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2435 00000100:00080000:1.0:1789763001.718937:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763001.718943:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789763001.718974:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2c000 x1876700024716032/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763001.718995:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763001.718998:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763001.718999:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763001.719002:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2435 00000100:00080000:1.0:1789763001.719004:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763001.719006:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789763001.719013:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763001.719022:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2ca80 x1876700024716160/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763001.719031:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763001.719033:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763001.719034:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763001.719039:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2435 00000100:00100000:2.0:1789763001.719040:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024716032:192.168.203.107@tcp:250:kworker.0 00000100:00080000:1.0:1789763001.719041:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763001.719048:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789763001.719065:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763001.719075:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2c700 x1876700024716288/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789763001.719104:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789763001.719136:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763001/real 1789763001] req@ffffa06c08e2c000 x1876700024716032/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763026 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763001.719150:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2c000 x1876700024716032/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763026 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763001.719161:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763001.719166:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789763001.719169:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024716032:192.168.203.107@tcp:250:kworker.0 00000100:00100000:2.0:1789763001.719192:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024716160:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789763001.719215:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789763001.719225:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763001/real 1789763001] req@ffffa06c08e2ca80 x1876700024716160/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763026 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763001.719250:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2ca80 x1876700024716160/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763026 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763001.719257:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763001.719259:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789763001.719268:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024716160:192.168.203.107@tcp:38:kworker.0 00000100:00100000:2.0:1789763001.719273:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024716288:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789763001.719290:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789763001.719297:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763001/real 1789763001] req@ffffa06c08e2c700 x1876700024716288/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763026 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763001.719305:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2c700 x1876700024716288/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763026 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763001.719310:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763001.719312:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789763001.719314:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024716288:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789763001.974967:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789763002.230894:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789763002.999054:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763003.254973:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763004.022999:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763004.278935:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763005.046999:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763005.302946:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763006.071029:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763006.326946:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789763006.838984:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763006.838992:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763006.838995:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763006.839003:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2440 00000100:00080000:2.0:1789763006.839012:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763006.839018:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789763006.839052:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e52a00 x1876700024716416/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763006.839079:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763006.839082:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763006.839083:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763006.839086:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2440 00000100:00080000:2.0:1789763006.839104:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763006.839108:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789763006.839115:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763006.839126:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e53100 x1876700024716544/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763006.839135:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763006.839137:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763006.839140:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763006.839147:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2440 00000100:00100000:3.0:1789763006.839150:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e52a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024716416:192.168.203.107@tcp:250:kworker.0 00000100:00080000:2.0:1789763006.839151:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763006.839163:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789763006.839168:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763006.839183:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e52d80 x1876700024716672/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789763006.839256:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763006.839305:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763006/real 1789763006] req@ffffa06c07e52a00 x1876700024716416/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763031 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763006.839321:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e52a00 x1876700024716416/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763031 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763006.839330:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763006.839334:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763006.839337:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e52a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024716416:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789763006.839361:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e53100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024716544:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763006.839377:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763006.839386:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763006/real 1789763006] req@ffffa06c07e53100 x1876700024716544/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763031 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763006.839395:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e53100 x1876700024716544/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763031 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763006.839425:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763006.839430:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763006.839433:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e53100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024716544:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789763006.839441:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e52d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024716672:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763006.839464:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763006.839473:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763006/real 1789763006] req@ffffa06c07e52d80 x1876700024716672/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763031 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763006.839481:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e52d80 x1876700024716672/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763031 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763006.839489:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763006.839493:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763006.839496:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e52d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024716672:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789763007.094945:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763007.350970:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763008.119009:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763008.374932:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763009.142979:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763009.398913:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763010.167001:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763010.422928:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789763011.190957:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763011.446920:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:1.0:1789763011.958947:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763011.958957:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789763011.958961:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763011.958972:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2445 00000100:00080000:1.0:1789763011.958983:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763011.958991:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789763011.959032:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2d180 x1876700024716800/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763011.959056:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763011.959060:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763011.959061:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763011.959065:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2445 00000100:00080000:1.0:1789763011.959067:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763011.959069:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789763011.959078:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763011.959091:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2c700 x1876700024716928/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763011.959100:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763011.959103:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763011.959104:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763011.959106:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2445 00000100:00100000:3.0:1789763011.959114:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024716800:192.168.203.107@tcp:250:kworker.0 00000100:00080000:1.0:1789763011.959122:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763011.959128:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789763011.959132:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763011.959141:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2ca80 x1876700024717056/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789763011.959213:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763011.959273:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763011/real 1789763011] req@ffffa06c08e2d180 x1876700024716800/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763036 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763011.959295:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2d180 x1876700024716800/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763036 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763011.959309:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763011.959314:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763011.959319:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024716800:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789763011.959327:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024716928:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763011.959351:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763011.959365:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763011/real 1789763011] req@ffffa06c08e2c700 x1876700024716928/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763036 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763011.959379:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2c700 x1876700024716928/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763036 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763011.959390:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763011.959394:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763011.959397:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024716928:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789763011.959425:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024717056:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763011.959450:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763011.959470:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763011/real 1789763011] req@ffffa06c08e2ca80 x1876700024717056/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763036 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763011.959484:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2ca80 x1876700024717056/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763036 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763011.959506:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763011.959510:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763011.959514:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024717056:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789763012.214973:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763012.470939:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763013.238967:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763013.494964:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789763014.263054:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763014.518901:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763015.287007:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763015.542979:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763016.311165:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763016.566953:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:1.0:1789763017.078938:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763017.078947:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789763017.078951:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763017.078960:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2450 00000100:00080000:1.0:1789763017.078969:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763017.078974:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789763017.079006:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2c000 x1876700024717184/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763017.079027:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763017.079030:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763017.079031:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763017.079034:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2450 00000100:00080000:1.0:1789763017.079037:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763017.079041:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789763017.079048:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763017.079058:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024717184:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789763017.079061:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2e300 x1876700024717312/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763017.079078:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763017.079096:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763017.079098:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763017.079103:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2450 00000100:00080000:1.0:1789763017.079109:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763017.079128:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789763017.079155:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763017.079178:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2ed80 x1876700024717440/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763017.079236:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024717312:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789763017.079302:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024717440:192.168.203.107@tcp:38:kworker.0 00010000:00010000:1.0:1789763017.335079:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763017.590976:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789763018.359050:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763018.614943:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763019.383043:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763019.638940:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000400:00000100:2.0:1789763020.151404:0:2331:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.107:988 00000400:00000100:2.0:1789763020.151490:0:2331:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.107@tcp at host 192.168.203.107 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1789763020.151519:0:2331:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.107@tcp 00000800:00000100:2.0:1789763020.151627:0:2331:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.107@tcp 00000800:00000100:2.0:1789763020.151666:0:2331:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.107@tcp 00000100:00100000:3.0:1789763020.152226:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763017/real 1789763020] req@ffffa06c08e2e300 x1876700024717312/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763042 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763020.152281:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2e300 x1876700024717312/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763042 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763020.152316:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763020.152625:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763020.152643:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024717312:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789763020.152663:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763017/real 1789763020] req@ffffa06c08e2ed80 x1876700024717440/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763042 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763020.152697:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2ed80 x1876700024717440/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763042 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763020.152721:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763020.152727:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763020.152732:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024717440:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789763020.152742:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763017/real 1789763020] req@ffffa06c08e2c000 x1876700024717184/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763042 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763020.152759:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2c000 x1876700024717184/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763042 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763020.152771:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763020.152778:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763020.152784:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024717184:192.168.203.107@tcp:250:kworker.0 00010000:00010000:1.0:1789763020.407078:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763020.662992:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789763021.431078:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763021.686996:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:3.0:1789763022.199163:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763022.199181:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763022.199188:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763022.199203:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2456 00000100:00080000:3.0:1789763022.199217:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763022.199247:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:3.0:1789763022.199302:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872fb80 x1876700024717568/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763022.199355:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763022.199362:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763022.199364:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763022.199371:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2456 00000100:00080000:3.0:1789763022.199379:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763022.199386:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789763022.199404:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763022.199436:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872f480 x1876700024717696/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763022.199482:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763022.199491:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763022.199495:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1789763022.199506:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024717568:192.168.203.107@tcp:250:kworker.0 00000100:00080000:3.0:1789763022.199507:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2456 00000100:00080000:3.0:1789763022.199520:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763022.199532:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789763022.199555:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763022.199605:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872ea00 x1876700024717824/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789763022.199714:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789763022.199800:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763022/real 1789763022] req@ffffa06c1872fb80 x1876700024717568/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763047 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763022.199986:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872fb80 x1876700024717568/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763047 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763022.200014:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763022.200025:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789763022.200068:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024717568:192.168.203.107@tcp:250:kworker.0 00000100:00100000:2.0:1789763022.200145:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024717696:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789763022.200202:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789763022.200234:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763022/real 1789763022] req@ffffa06c1872f480 x1876700024717696/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763047 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763022.200262:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872f480 x1876700024717696/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763047 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763022.200280:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763022.200287:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789763022.200294:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024717696:192.168.203.107@tcp:38:kworker.0 00000100:00100000:2.0:1789763022.200309:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024717824:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789763022.200354:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789763022.200380:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763022/real 1789763022] req@ffffa06c1872ea00 x1876700024717824/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763047 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763022.200412:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872ea00 x1876700024717824/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763047 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763022.200452:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763022.200459:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789763022.200463:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024717824:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789763022.455104:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763022.711004:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789763023.479172:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763023.734944:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789763024.503051:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763024.759003:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763025.527097:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763025.782958:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763026.551038:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763026.806968:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789763027.319058:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763027.319072:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763027.319078:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763027.319090:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2461 00000100:00080000:2.0:1789763027.319104:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763027.319123:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789763027.319168:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50700 x1876700024717952/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763027.319206:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763027.319212:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763027.319215:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763027.319220:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2461 00000100:00080000:2.0:1789763027.319224:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763027.319229:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789763027.319241:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763027.319258:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50a80 x1876700024718080/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763027.319276:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763027.319283:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:3.0:1789763027.319290:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024717952:192.168.203.107@tcp:250:kworker.0 00000100:00080000:2.0:1789763027.319302:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763027.319313:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2461 00000100:00080000:2.0:1789763027.319319:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763027.319324:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789763027.319332:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763027.319349:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e53480 x1876700024718208/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789763027.319421:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763027.319514:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763027/real 1789763027] req@ffffa06c07e50700 x1876700024717952/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763052 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763027.319540:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50700 x1876700024717952/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763052 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763027.319555:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763027.319560:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763027.319565:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024717952:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789763027.319596:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024718080:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763027.319655:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763027.319690:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763027/real 1789763027] req@ffffa06c07e50a80 x1876700024718080/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763052 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763027.319716:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50a80 x1876700024718080/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763052 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763027.319734:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763027.319741:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763027.319749:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024718080:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789763027.319774:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e53480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024718208:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763027.319828:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763027.319899:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763027/real 1789763027] req@ffffa06c07e53480 x1876700024718208/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763052 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763027.319928:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e53480 x1876700024718208/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763052 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763027.319945:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763027.319952:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763027.319959:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e53480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024718208:192.168.203.107@tcp:38:kworker.0 00010000:00010000:0.0:1789763027.575095:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763027.830945:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763028.599194:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763028.855064:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763029.623077:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763029.878996:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789763030.647139:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763030.902975:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763031.671086:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763031.926973:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789763032.438972:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763032.438985:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763032.438991:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763032.439033:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2466 00000100:00080000:2.0:1789763032.439048:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763032.439056:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789763032.439119:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51c00 x1876700024718336/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763032.439157:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763032.439164:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763032.439168:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763032.439176:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2466 00000100:00080000:2.0:1789763032.439183:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763032.439190:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789763032.439205:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763032.439229:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e53800 x1876700024718464/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763032.439242:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024718336:192.168.203.107@tcp:250:kworker.0 00000100:00080000:2.0:1789763032.439280:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763032.439285:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763032.439288:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763032.439294:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2466 00000100:00080000:2.0:1789763032.439332:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763032.439340:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000800:00000100:3.0:1789763032.439343:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000008:00100000:2.0:1789763032.439351:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763032.439373:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e52680 x1876700024718592/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763032.439403:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763032/real 1789763032] req@ffffa06c07e51c00 x1876700024718336/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763057 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763032.439433:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51c00 x1876700024718336/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763057 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763032.439452:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763032.439473:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763032.439481:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024718336:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789763032.439520:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e53800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024718464:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763032.439572:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763032.439614:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763032/real 1789763032] req@ffffa06c07e53800 x1876700024718464/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763057 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763032.439641:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e53800 x1876700024718464/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763057 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763032.439657:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763032.439663:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763032.439671:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e53800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024718464:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789763032.439683:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e52680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024718592:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763032.439727:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763032.439755:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763032/real 1789763032] req@ffffa06c07e52680 x1876700024718592/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763057 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763032.440196:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e52680 x1876700024718592/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763057 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763032.440214:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763032.440221:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763032.440228:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e52680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024718592:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789763032.695072:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763032.950963:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789763033.719146:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763033.975085:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763034.743045:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763034.998919:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763035.767015:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789763036.022922:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789763036.790954:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789763037.046944:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:1.0:1789763037.559039:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763037.559056:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789763037.559062:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763037.559078:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2471 00000100:00080000:1.0:1789763037.559093:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763037.559106:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789763037.559162:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2d500 x1876700024718720/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763037.559201:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763037.559207:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763037.559224:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763037.559230:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2471 00000100:00080000:1.0:1789763037.559237:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763037.559243:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789763037.559257:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763037.559277:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2c000 x1876700024718848/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763037.559294:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1789763037.559299:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024718720:192.168.203.107@tcp:250:kworker.0 00000100:00080000:1.0:1789763037.559300:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763037.559307:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763037.559320:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2471 00000100:00080000:1.0:1789763037.559324:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763037.559329:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789763037.559336:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763037.559351:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2ed80 x1876700024718976/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789763037.559409:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763037.559462:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763037/real 1789763037] req@ffffa06c08e2d500 x1876700024718720/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763062 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763037.559485:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2d500 x1876700024718720/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763062 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763037.559499:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763037.559504:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763037.559509:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024718720:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789763037.559537:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024718848:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763037.559562:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763037.559577:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763037/real 1789763037] req@ffffa06c08e2c000 x1876700024718848/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763062 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763037.559592:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2c000 x1876700024718848/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763062 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763037.559603:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763037.559607:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763037.559611:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024718848:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789763037.559633:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024718976:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763037.559659:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763037.559672:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763037/real 1789763037] req@ffffa06c08e2ed80 x1876700024718976/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763062 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763037.559687:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2ed80 x1876700024718976/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763062 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763037.559697:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763037.559700:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763037.559705:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024718976:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789763037.815250:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789763038.070955:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763038.839173:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789763039.094941:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763039.863010:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789763040.118932:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789763040.887088:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789763041.142916:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789763041.911160:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789763042.167013:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:1.0:1789763042.679013:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763042.679022:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789763042.679025:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763042.679034:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2476 00000100:00080000:1.0:1789763042.679043:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763042.679049:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789763042.679082:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2e300 x1876700024719104/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763042.679120:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763042.679123:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763042.679124:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763042.679127:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2476 00000100:00080000:1.0:1789763042.679130:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763042.679132:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789763042.679139:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763042.679149:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2ca80 x1876700024719232/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763042.679158:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763042.679161:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763042.679162:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763042.679164:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2476 00000100:00080000:1.0:1789763042.679167:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1789763042.679171:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024719104:192.168.203.107@tcp:250:kworker.0 00000100:00080000:1.0:1789763042.679172:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789763042.679179:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763042.679186:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2c700 x1876700024719360/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789763042.679231:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763042.679266:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763042/real 1789763042] req@ffffa06c08e2e300 x1876700024719104/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763067 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763042.679279:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2e300 x1876700024719104/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763067 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763042.679287:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763042.679290:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763042.679293:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024719104:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789763042.679299:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024719232:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763042.679331:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763042.679346:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763042/real 1789763042] req@ffffa06c08e2ca80 x1876700024719232/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763067 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763042.679360:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2ca80 x1876700024719232/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763067 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763042.679367:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763042.679370:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763042.679372:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024719232:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789763042.679395:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024719360:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763042.679419:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763042.679444:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763042/real 1789763042] req@ffffa06c08e2c700 x1876700024719360/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763067 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763042.679453:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2c700 x1876700024719360/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763067 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763042.679459:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763042.679461:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763042.679463:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024719360:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789763042.934975:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789763043.190996:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789763043.958983:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763044.214995:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789763044.983148:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763045.239054:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763046.006970:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763046.262957:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789763047.031120:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763047.286958:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:1.0:1789763047.799001:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763047.799014:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789763047.799019:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763047.799053:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2481 00000100:00080000:1.0:1789763047.799067:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763047.799076:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789763047.799120:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2d180 x1876700024719488/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763047.799153:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763047.799157:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763047.799160:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763047.799165:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2481 00000100:00080000:1.0:1789763047.799168:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763047.799172:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789763047.799183:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763047.799200:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2c700 x1876700024719616/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763047.799216:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1789763047.799220:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024719488:192.168.203.107@tcp:250:kworker.0 00000100:00080000:1.0:1789763047.799222:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763047.799229:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763047.799241:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2481 00000100:00080000:1.0:1789763047.799247:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763047.799256:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789763047.799266:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763047.799290:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2ca80 x1876700024719744/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789763047.799313:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763047.799373:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763047/real 1789763047] req@ffffa06c08e2d180 x1876700024719488/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763072 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763047.799397:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2d180 x1876700024719488/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763072 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763047.799409:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763047.799414:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763047.799419:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024719488:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789763047.799446:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024719616:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763047.799470:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763047.799484:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763047/real 1789763047] req@ffffa06c08e2c700 x1876700024719616/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763072 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763047.799499:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2c700 x1876700024719616/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763072 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763047.799509:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763047.799513:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763047.799517:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024719616:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789763047.799524:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024719744:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763047.799573:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763047.799597:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763047/real 1789763047] req@ffffa06c08e2ca80 x1876700024719744/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763072 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763047.799651:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2ca80 x1876700024719744/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763072 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763047.799662:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763047.799666:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763047.799670:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024719744:192.168.203.107@tcp:38:kworker.0 00010000:00010000:1.0:1789763048.055063:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763048.310958:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763049.079041:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763049.335023:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789763050.103031:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763050.359011:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789763051.127098:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763051.382979:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763052.151030:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763052.407004:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:1.0:1789763052.919008:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763052.919020:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789763052.919025:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763052.919038:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2486 00000100:00080000:1.0:1789763052.919051:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763052.919060:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789763052.919111:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2e300 x1876700024719872/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763052.919148:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763052.919153:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763052.919156:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763052.919160:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2486 00000100:00080000:1.0:1789763052.919165:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763052.919169:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789763052.919180:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763052.919195:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2ed80 x1876700024720000/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763052.919211:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763052.919216:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763052.919219:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763052.919224:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2486 00000100:00100000:3.0:1789763052.919251:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024719872:192.168.203.107@tcp:250:kworker.0 00000100:00080000:1.0:1789763052.919252:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763052.919269:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789763052.919277:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763052.919298:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2c000 x1876700024720128/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789763052.919350:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763052.919422:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763052/real 1789763052] req@ffffa06c08e2e300 x1876700024719872/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763077 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763052.919446:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2e300 x1876700024719872/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763077 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763052.919467:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763052.919475:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763052.919483:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024719872:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789763052.919497:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024720000:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763052.919558:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763052.919577:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763052/real 1789763052] req@ffffa06c08e2ed80 x1876700024720000/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763077 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763052.919594:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2ed80 x1876700024720000/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763077 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763052.919605:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763052.919610:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763052.919614:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024720000:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789763052.919645:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024720128:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763052.919667:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763052.919679:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763052/real 1789763052] req@ffffa06c08e2c000 x1876700024720128/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763077 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763052.919694:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2c000 x1876700024720128/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763077 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763052.919704:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763052.919707:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763052.919711:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024720128:192.168.203.107@tcp:38:kworker.0 00010000:00010000:1.0:1789763053.175054:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763053.430978:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789763054.199174:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763054.454992:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789763055.223037:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763055.478961:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789763056.247000:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763056.502950:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789763057.271095:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763057.527015:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:1.0:1789763058.039035:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763058.039047:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789763058.039053:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763058.039067:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2491 00000100:00080000:1.0:1789763058.039097:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763058.039106:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789763058.039148:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2d500 x1876700024720256/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763058.039175:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763058.039179:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763058.039181:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763058.039185:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2491 00000100:00080000:1.0:1789763058.039189:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763058.039193:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789763058.039202:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763058.039216:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2c000 x1876700024720384/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763058.039260:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024720256:192.168.203.107@tcp:250:kworker.0 00000100:00080000:1.0:1789763058.039281:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763058.039292:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763058.039296:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763058.039306:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2491 00000100:00080000:1.0:1789763058.039314:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763058.039323:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789763058.039333:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763058.039357:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2ed80 x1876700024720512/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789763058.039375:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763058.039429:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763058/real 1789763058] req@ffffa06c08e2d500 x1876700024720256/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763083 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763058.039468:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2d500 x1876700024720256/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763083 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763058.039480:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763058.039484:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763058.039488:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024720256:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789763058.039512:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024720384:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763058.039554:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763058.039574:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763058/real 1789763058] req@ffffa06c08e2c000 x1876700024720384/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763083 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763058.039588:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2c000 x1876700024720384/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763083 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763058.039615:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763058.039631:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763058.039635:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024720384:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789763058.039642:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024720512:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763058.039663:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763058.039674:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763058/real 1789763058] req@ffffa06c08e2ed80 x1876700024720512/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763083 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763058.039716:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2ed80 x1876700024720512/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763083 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763058.039726:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763058.039730:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763058.039734:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024720512:192.168.203.107@tcp:38:kworker.0 00010000:00010000:0.0:1789763058.295080:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763058.550964:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789763059.319071:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763059.574948:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789763060.343092:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763060.598953:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763061.367026:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763061.622954:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789763062.391183:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763062.646992:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789763063.159039:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763063.159056:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763063.159063:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763063.159081:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2496 00000100:00080000:2.0:1789763063.159100:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763063.159113:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789763063.159207:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50380 x1876700024720640/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763063.159266:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763063.159275:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763063.159300:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763063.159313:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2496 00000100:00080000:2.0:1789763063.159323:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763063.159334:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789763063.159354:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763063.159364:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024720640:192.168.203.107@tcp:250:kworker.0 00000100:00100000:2.0:1789763063.159392:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e53b80 x1876700024720768/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763063.159423:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763063.159431:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763063.159436:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763063.159445:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2496 00000100:00080000:2.0:1789763063.159480:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763063.159491:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000800:00000100:3.0:1789763063.159503:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000008:00100000:2.0:1789763063.159510:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763063.159543:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51f80 x1876700024720896/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763063.159594:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763063/real 1789763063] req@ffffa06c07e50380 x1876700024720640/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763088 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763063.159625:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50380 x1876700024720640/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763088 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763063.159641:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763063.159647:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763063.159653:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024720640:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789763063.159690:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e53b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024720768:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763063.159724:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763063.159757:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763063/real 1789763063] req@ffffa06c07e53b80 x1876700024720768/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763088 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763063.159774:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e53b80 x1876700024720768/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763088 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763063.159820:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763063.159828:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763063.159836:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e53b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024720768:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789763063.159914:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024720896:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763063.159952:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763063.159970:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763063/real 1789763063] req@ffffa06c07e51f80 x1876700024720896/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763088 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763063.159987:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51f80 x1876700024720896/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763088 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763063.159998:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763063.160002:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763063.160007:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024720896:192.168.203.107@tcp:38:kworker.0 00010000:00010000:0.0:1789763063.415154:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763063.670962:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789763064.439020:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763064.694971:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789763065.463130:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763065.718965:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789763066.487078:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763066.742952:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789763067.511136:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763067.766967:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789763068.278992:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763068.279007:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763068.279013:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763068.279028:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2502 00000100:00080000:0.0:1789763068.279061:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763068.279071:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789763068.279119:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d2300 x1876700024721024/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763068.279162:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763068.279168:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763068.279170:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763068.279178:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2502 00000100:00080000:0.0:1789763068.279187:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763068.279194:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789763068.279208:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763068.279231:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d3b80 x1876700024721152/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763068.279244:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024721024:192.168.203.107@tcp:250:kworker.0 00000100:00080000:0.0:1789763068.279254:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763068.279261:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763068.279264:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763068.279270:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2502 00000100:00080000:0.0:1789763068.279277:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763068.279282:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789763068.279289:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763068.279309:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0000 x1876700024721280/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789763068.279345:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789763068.279416:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763068/real 1789763068] req@ffffa06c091d2300 x1876700024721024/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763093 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763068.279458:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d2300 x1876700024721024/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763093 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763068.279497:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763068.279506:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789763068.279516:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024721024:192.168.203.107@tcp:250:kworker.0 00000100:00100000:2.0:1789763068.279552:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024721152:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789763068.279591:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789763068.279610:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763068/real 1789763068] req@ffffa06c091d3b80 x1876700024721152/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763093 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763068.279631:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d3b80 x1876700024721152/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763093 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763068.279643:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763068.279648:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789763068.279653:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024721152:192.168.203.107@tcp:38:kworker.0 00000100:00100000:2.0:1789763068.279662:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024721280:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789763068.279686:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789763068.279701:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763068/real 1789763068] req@ffffa06c091d0000 x1876700024721280/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763093 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763068.279717:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0000 x1876700024721280/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763093 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763068.279728:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763068.279735:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789763068.279767:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024721280:192.168.203.107@tcp:38:kworker.0 00010000:00010000:2.0:1789763068.535025:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763068.790958:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763069.559080:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763069.814973:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789763070.583066:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763070.838976:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789763071.607034:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763071.862950:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763072.630977:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763072.886950:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:1.0:1789763073.399066:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763073.399079:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789763073.399084:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763073.399096:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2507 00000100:00080000:1.0:1789763073.399127:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763073.399134:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789763073.399186:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2e300 x1876700024721408/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763073.399218:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763073.399222:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763073.399224:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763073.399228:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2507 00000100:00080000:1.0:1789763073.399251:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763073.399258:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789763073.399268:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763073.399284:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2ca80 x1876700024721536/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763073.399297:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763073.399306:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763073.399309:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1789763073.399309:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024721408:192.168.203.107@tcp:250:kworker.0 00000100:00080000:1.0:1789763073.399332:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2507 00000100:00080000:1.0:1789763073.399335:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763073.399340:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789763073.399344:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763073.399357:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2c700 x1876700024721664/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789763073.399460:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789763073.399544:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763073/real 1789763073] req@ffffa06c08e2e300 x1876700024721408/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763098 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763073.399576:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2e300 x1876700024721408/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763098 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763073.399593:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763073.399599:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789763073.399605:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024721408:192.168.203.107@tcp:250:kworker.0 00000100:00100000:2.0:1789763073.399645:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024721536:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789763073.399685:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789763073.399736:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763073/real 1789763073] req@ffffa06c08e2ca80 x1876700024721536/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763098 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763073.399764:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2ca80 x1876700024721536/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763098 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763073.399784:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763073.399792:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789763073.399800:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024721536:192.168.203.107@tcp:38:kworker.0 00000100:00100000:2.0:1789763073.399810:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024721664:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789763073.399887:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789763073.399924:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763073/real 1789763073] req@ffffa06c08e2c700 x1876700024721664/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763098 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763073.399941:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2c700 x1876700024721664/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763098 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763073.399952:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763073.399956:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789763073.399961:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024721664:192.168.203.107@tcp:38:kworker.0 00010000:00010000:0.0:1789763073.655231:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763073.910949:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789763074.678996:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763074.934918:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763075.703028:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763075.958978:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789763076.727053:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763076.982969:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789763077.751017:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763078.006964:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789763078.519001:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763078.519014:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763078.519020:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763078.519033:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2512 00000100:00080000:2.0:1789763078.519047:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763078.519057:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789763078.519112:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50e00 x1876700024721792/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763078.519171:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763078.519177:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763078.519180:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763078.519186:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2512 00000100:00080000:2.0:1789763078.519192:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763078.519198:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789763078.519212:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763078.519257:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024721792:192.168.203.107@tcp:250:kworker.0 00000100:00100000:2.0:1789763078.519269:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51500 x1876700024721920/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763078.519292:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763078.519297:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763078.519299:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763078.519305:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2512 00000100:00080000:2.0:1789763078.519311:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763078.519317:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789763078.519324:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763078.519339:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50000 x1876700024722048/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789763078.519366:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763078.519448:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763078/real 1789763078] req@ffffa06c07e50e00 x1876700024721792/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763103 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763078.519491:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50e00 x1876700024721792/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763103 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763078.519506:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763078.519512:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763078.519517:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024721792:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789763078.519547:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024721920:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763078.519591:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763078.519609:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763078/real 1789763078] req@ffffa06c07e51500 x1876700024721920/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763103 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763078.519626:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51500 x1876700024721920/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763103 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763078.519658:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763078.519686:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763078.519692:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024721920:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789763078.519702:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024722048:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763078.519743:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763078.519760:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763078/real 1789763078] req@ffffa06c07e50000 x1876700024722048/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763103 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763078.519774:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50000 x1876700024722048/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763103 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763078.519785:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763078.519789:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763078.519793:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024722048:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789763078.775145:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763079.030984:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789763079.799071:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763080.054968:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789763080.823076:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763081.078962:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763081.847011:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763082.102955:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789763082.871050:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763083.127007:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:1.0:1789763083.638990:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763083.639002:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789763083.639006:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763083.639018:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2517 00000100:00080000:1.0:1789763083.639031:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763083.639039:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789763083.639103:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2d180 x1876700024722176/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763083.639148:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763083.639170:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763083.639174:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763083.639181:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2517 00000100:00080000:1.0:1789763083.639186:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763083.639192:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:3.0:1789763083.639202:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024722176:192.168.203.107@tcp:250:kworker.0 00000008:00100000:1.0:1789763083.639206:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763083.639244:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2c700 x1876700024722304/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763083.639255:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763083.639259:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763083.639260:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763083.639264:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2517 00000100:00080000:1.0:1789763083.639267:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763083.639270:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789763083.639274:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763083.639283:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2ca80 x1876700024722432/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763083.639407:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024722304:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789763083.639442:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024722432:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789763083.895036:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763084.150965:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789763084.919066:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763085.174979:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789763085.943081:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763086.198952:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000400:00000100:0.0:1789763086.711235:0:2328:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.107:988 00000400:00000100:0.0:1789763086.711305:0:2328:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.107@tcp at host 192.168.203.107 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1789763086.711335:0:2328:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.107@tcp 00000800:00000100:0.0:1789763086.711413:0:2328:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.107@tcp 00000800:00000100:0.0:1789763086.711444:0:2328:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.107@tcp 00000100:00100000:3.0:1789763086.711491:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763083/real 1789763086] req@ffffa06c08e2c700 x1876700024722304/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763108 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763086.711540:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2c700 x1876700024722304/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763108 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763086.711577:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763086.711588:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763086.711597:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024722304:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789763086.711634:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763083/real 1789763086] req@ffffa06c08e2ca80 x1876700024722432/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763108 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763086.711674:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2ca80 x1876700024722432/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763108 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763086.711692:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763086.711699:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763086.711706:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024722432:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789763086.711758:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763083/real 1789763086] req@ffffa06c08e2d180 x1876700024722176/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763108 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763086.711789:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2d180 x1876700024722176/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763108 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763086.711834:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763086.711885:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763086.711895:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024722176:192.168.203.107@tcp:250:kworker.0 00010000:00010000:3.0:1789763086.967017:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763087.222951:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789763087.991053:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763088.246964:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:1.0:1789763088.759017:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763088.759056:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789763088.759062:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763088.759078:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2522 00000100:00080000:1.0:1789763088.759095:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763088.759108:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789763088.759184:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2e300 x1876700024722560/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763088.759247:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763088.759256:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763088.759260:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763088.759269:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2522 00000100:00080000:1.0:1789763088.759279:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763088.759303:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789763088.759318:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763088.759340:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024722560:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789763088.759351:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2ed80 x1876700024722688/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763088.759396:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763088.759409:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763088.759412:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763088.759419:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2522 00000800:00000100:2.0:1789763088.759429:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00080000:1.0:1789763088.759433:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763088.759445:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789763088.759471:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763088.759500:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2c000 x1876700024722816/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763088.759535:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763088/real 1789763088] req@ffffa06c08e2e300 x1876700024722560/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763113 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763088.759584:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2e300 x1876700024722560/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763113 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763088.759627:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763088.759646:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789763088.759669:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024722560:192.168.203.107@tcp:250:kworker.0 00000100:00100000:2.0:1789763088.759743:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024722688:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789763088.759791:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789763088.759818:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763088/real 1789763088] req@ffffa06c08e2ed80 x1876700024722688/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763113 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763088.759889:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2ed80 x1876700024722688/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763113 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763088.759911:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763088.759919:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789763088.759927:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024722688:192.168.203.107@tcp:38:kworker.0 00000100:00100000:2.0:1789763088.759941:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024722816:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789763088.760005:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789763088.760044:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763088/real 1789763088] req@ffffa06c08e2c000 x1876700024722816/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763113 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763088.760068:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2c000 x1876700024722816/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763113 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763088.760082:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763088.760089:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789763088.760095:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024722816:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789763089.015069:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763089.270981:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763090.039112:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763090.294953:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763091.062965:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763091.318958:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763092.087022:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763092.342923:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763093.111115:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763093.366954:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:1.0:1789763093.878968:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763093.878981:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789763093.878986:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763093.878999:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2527 00000100:00080000:1.0:1789763093.879046:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763093.879055:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789763093.879099:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2d500 x1876700024722944/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763093.879151:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763093.879159:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763093.879163:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763093.879172:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2527 00000100:00080000:1.0:1789763093.879183:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763093.879192:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789763093.879210:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763093.879238:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2c000 x1876700024723072/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763093.879253:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024722944:192.168.203.107@tcp:250:kworker.0 00000100:00080000:1.0:1789763093.879265:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763093.879272:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763093.879275:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763093.879281:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2527 00000100:00080000:1.0:1789763093.879286:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763093.879292:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789763093.879299:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763093.879315:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2ed80 x1876700024723200/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789763093.879411:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789763093.879542:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763093/real 1789763093] req@ffffa06c08e2d500 x1876700024722944/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763118 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763093.879607:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2d500 x1876700024722944/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763118 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763093.879625:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763093.879631:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789763093.879637:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024722944:192.168.203.107@tcp:250:kworker.0 00000100:00100000:2.0:1789763093.879682:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024723072:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789763093.879720:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789763093.879739:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763093/real 1789763093] req@ffffa06c08e2c000 x1876700024723072/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763118 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763093.879756:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2c000 x1876700024723072/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763118 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763093.879767:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763093.879772:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789763093.879776:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024723072:192.168.203.107@tcp:38:kworker.0 00000100:00100000:2.0:1789763093.879784:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024723200:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789763093.879806:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789763093.879818:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763093/real 1789763093] req@ffffa06c08e2ed80 x1876700024723200/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763118 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763093.879833:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2ed80 x1876700024723200/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763118 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763093.879906:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763093.879911:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789763093.879916:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024723200:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789763094.135034:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763094.390921:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763095.159064:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763095.414995:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763096.183061:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763096.438959:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763097.207140:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763097.462979:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763098.231040:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763098.486974:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:3.0:1789763098.999052:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763098.999078:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763098.999107:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763098.999121:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2532 00000100:00080000:3.0:1789763098.999134:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763098.999143:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:3.0:1789763098.999188:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872ed80 x1876700024723328/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763098.999228:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763098.999234:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763098.999236:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763098.999241:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2532 00000100:00080000:3.0:1789763098.999246:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763098.999250:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789763098.999262:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763098.999278:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872dc00 x1876700024723456/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763098.999293:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763098.999298:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763098.999301:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763098.999306:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2532 00000100:00080000:3.0:1789763098.999310:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763098.999314:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789763098.999321:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763098.999330:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024723328:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789763098.999342:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872e680 x1876700024723584/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789763098.999458:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789763098.999542:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763098/real 1789763098] req@ffffa06c1872ed80 x1876700024723328/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763123 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763098.999598:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872ed80 x1876700024723328/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763123 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763098.999623:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763098.999632:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789763098.999638:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024723328:192.168.203.107@tcp:250:kworker.0 00000100:00100000:2.0:1789763098.999651:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024723456:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789763098.999689:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789763098.999707:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763098/real 1789763098] req@ffffa06c1872dc00 x1876700024723456/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763123 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763098.999724:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872dc00 x1876700024723456/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763123 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763098.999736:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763098.999741:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789763098.999745:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024723456:192.168.203.107@tcp:38:kworker.0 00000100:00100000:2.0:1789763098.999804:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024723584:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789763098.999834:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789763098.999847:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763098/real 1789763098] req@ffffa06c1872e680 x1876700024723584/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763123 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763098.999920:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872e680 x1876700024723584/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763123 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763098.999932:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763098.999936:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789763098.999940:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024723584:192.168.203.107@tcp:38:kworker.0 00010000:00010000:2.0:1789763099.255099:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763099.510971:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763100.279014:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763100.535024:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763101.303058:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763101.559007:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789763102.327018:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763102.582938:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763103.351096:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763103.606975:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789763104.119021:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763104.119037:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763104.119044:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763104.119060:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2537 00000100:00080000:2.0:1789763104.119078:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763104.119090:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789763104.119153:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51180 x1876700024723712/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763104.119200:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763104.119205:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763104.119208:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763104.119214:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2537 00000100:00080000:2.0:1789763104.119245:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763104.119252:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789763104.119268:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763104.119281:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024723712:192.168.203.107@tcp:250:kworker.0 00000100:00100000:2.0:1789763104.119297:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e52300 x1876700024723840/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763104.119320:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763104.119326:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763104.119328:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763104.119334:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2537 00000100:00080000:2.0:1789763104.119339:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763104.119345:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789763104.119354:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763104.119371:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51880 x1876700024723968/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789763104.119397:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763104.119463:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763104/real 1789763104] req@ffffa06c07e51180 x1876700024723712/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763129 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763104.119524:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51180 x1876700024723712/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763129 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763104.119542:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763104.119549:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763104.119563:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024723712:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789763104.119599:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e52300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024723840:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763104.119636:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763104.119661:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763104/real 1789763104] req@ffffa06c07e52300 x1876700024723840/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763129 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763104.119680:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e52300 x1876700024723840/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763129 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763104.119718:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763104.119725:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763104.119734:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e52300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024723840:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789763104.119746:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024723968:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763104.119780:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763104.119794:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763104/real 1789763104] req@ffffa06c07e51880 x1876700024723968/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763129 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763104.119808:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51880 x1876700024723968/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763129 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763104.119826:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763104.119831:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763104.119834:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024723968:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789763104.375131:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763104.630988:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789763105.399153:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763105.655004:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763106.422995:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763106.678966:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763107.447075:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763107.702974:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789763108.471026:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763108.726971:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:1.0:1789763109.239075:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763109.239106:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789763109.239111:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763109.239123:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2543 00000100:00080000:1.0:1789763109.239137:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763109.239156:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789763109.239216:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2e300 x1876700024724096/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763109.239262:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763109.239267:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763109.239270:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763109.239275:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2543 00000100:00080000:1.0:1789763109.239279:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763109.239283:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789763109.239294:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763109.239309:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2d180 x1876700024724224/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763109.239323:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763109.239327:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763109.239329:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763109.239333:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2543 00000100:00080000:1.0:1789763109.239336:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763109.239340:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789763109.239345:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763109.239356:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2ca80 x1876700024724352/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763109.239411:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024724096:192.168.203.107@tcp:250:kworker.0 00000800:00000100:3.0:1789763109.239539:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763109.239601:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763109/real 1789763109] req@ffffa06c08e2e300 x1876700024724096/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763134 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763109.239640:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2e300 x1876700024724096/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763134 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763109.239654:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763109.239660:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763109.239665:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024724096:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789763109.239676:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024724224:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763109.239714:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763109.239729:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763109/real 1789763109] req@ffffa06c08e2d180 x1876700024724224/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763134 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763109.239744:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2d180 x1876700024724224/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763134 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763109.239755:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763109.239758:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763109.239762:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024724224:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789763109.239770:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024724352:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763109.239789:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763109.239801:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763109/real 1789763109] req@ffffa06c08e2ca80 x1876700024724352/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763134 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763109.239815:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2ca80 x1876700024724352/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763134 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763109.239841:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763109.239884:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763109.239889:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024724352:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789763109.495026:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763109.751036:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763110.519128:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763110.774976:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789763111.543081:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763111.798974:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763112.567025:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763112.822983:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763113.591053:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763113.847044:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789763114.359031:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763114.359056:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763114.359061:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763114.359074:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2548 00000100:00080000:0.0:1789763114.359088:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763114.359097:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789763114.359143:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d1500 x1876700024724480/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763114.359181:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763114.359186:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763114.359189:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763114.359194:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2548 00000100:00080000:0.0:1789763114.359198:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763114.359202:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789763114.359215:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763114.359256:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0e00 x1876700024724608/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763114.359277:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763114.359285:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:3.0:1789763114.359288:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024724480:192.168.203.107@tcp:250:kworker.0 00000100:00080000:0.0:1789763114.359289:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763114.359317:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2548 00000100:00080000:0.0:1789763114.359329:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763114.359339:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789763114.359346:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763114.359362:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d2a00 x1876700024724736/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789763114.359500:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763114.359579:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763114/real 1789763114] req@ffffa06c091d1500 x1876700024724480/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763139 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763114.359611:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d1500 x1876700024724480/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763139 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763114.359625:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763114.359630:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763114.359634:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024724480:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789763114.359668:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024724608:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763114.359694:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763114.359748:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763114/real 1789763114] req@ffffa06c091d0e00 x1876700024724608/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763139 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763114.359765:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0e00 x1876700024724608/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763139 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763114.359778:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763114.359783:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763114.359787:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024724608:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789763114.359796:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024724736:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763114.359822:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763114.359835:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763114/real 1789763114] req@ffffa06c091d2a00 x1876700024724736/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763139 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763114.359850:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d2a00 x1876700024724736/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763139 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763114.359907:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763114.359912:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763114.359916:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024724736:192.168.203.107@tcp:38:kworker.0 00010000:00010000:1.0:1789763114.615140:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763114.870962:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763115.638940:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763115.894906:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763116.663020:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763116.918964:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763117.687011:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763117.942917:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763118.711110:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763118.966979:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:1.0:1789763119.479087:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763119.479106:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789763119.479115:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763119.479134:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2553 00000100:00080000:1.0:1789763119.479153:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763119.479184:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789763119.479234:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2c700 x1876700024724864/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763119.479272:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763119.479277:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763119.479280:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763119.479286:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2553 00000100:00080000:1.0:1789763119.479293:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763119.479300:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789763119.479328:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763119.479352:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2ca80 x1876700024724992/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763119.479353:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024724864:192.168.203.107@tcp:250:kworker.0 00000100:00080000:1.0:1789763119.479404:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763119.479427:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763119.479430:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763119.479439:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2553 00000100:00080000:1.0:1789763119.479470:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763119.479477:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789763119.479486:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763119.479524:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2d180 x1876700024725120/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789763119.479524:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763119.479606:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763119/real 1789763119] req@ffffa06c08e2c700 x1876700024724864/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763144 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763119.479631:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2c700 x1876700024724864/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763144 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763119.479647:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763119.479652:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763119.479658:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024724864:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789763119.479718:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024724992:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763119.479769:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763119.479788:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763119/real 1789763119] req@ffffa06c08e2ca80 x1876700024724992/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763144 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763119.479804:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2ca80 x1876700024724992/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763144 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763119.479816:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763119.479820:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763119.479825:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024724992:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789763119.479833:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024725120:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763119.479905:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763119.479921:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763119/real 1789763119] req@ffffa06c08e2d180 x1876700024725120/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763144 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763119.479957:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2d180 x1876700024725120/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763144 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763119.479982:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763119.479989:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763119.479995:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024725120:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789763119.735055:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763119.990971:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763120.759041:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763121.014985:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763121.783021:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763122.038941:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789763122.807091:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763123.062961:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763123.831060:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763124.086946:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789763124.599059:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763124.599072:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763124.599077:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763124.599107:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2558 00000100:00080000:2.0:1789763124.599122:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763124.599131:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789763124.599180:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51880 x1876700024725248/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763124.599250:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763124.599274:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763124.599277:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763124.599283:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2558 00000100:00080000:2.0:1789763124.599289:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763124.599294:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789763124.599307:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763124.599352:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e52300 x1876700024725376/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763124.599372:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763124.599379:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763124.599390:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1789763124.599393:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024725248:192.168.203.107@tcp:250:kworker.0 00000100:00080000:2.0:1789763124.599399:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2558 00000100:00080000:2.0:1789763124.599412:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763124.599420:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789763124.599426:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763124.599453:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51180 x1876700024725504/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789763124.599527:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763124.599602:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763124/real 1789763124] req@ffffa06c07e51880 x1876700024725248/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763149 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763124.599625:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51880 x1876700024725248/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763149 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763124.599638:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763124.599644:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763124.599651:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024725248:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789763124.599724:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e52300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024725376:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763124.599764:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763124.599781:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763124/real 1789763124] req@ffffa06c07e52300 x1876700024725376/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763149 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763124.599794:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e52300 x1876700024725376/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763149 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763124.599804:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763124.599808:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763124.599811:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e52300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024725376:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789763124.599818:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024725504:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763124.599837:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763124.599846:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763124/real 1789763124] req@ffffa06c07e51180 x1876700024725504/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763149 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763124.599857:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51180 x1876700024725504/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763149 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763124.599898:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763124.599901:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763124.599904:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024725504:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789763124.855105:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789763125.110988:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763125.878983:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763126.134971:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763126.903056:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763127.158968:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763127.926976:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763128.183019:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763128.950996:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763129.206985:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789763129.719026:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763129.719040:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763129.719046:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763129.719064:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2563 00000100:00080000:0.0:1789763129.719082:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763129.719094:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789763129.719172:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d2680 x1876700024725632/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763129.719217:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763129.719224:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763129.719227:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763129.719235:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2563 00000100:00080000:0.0:1789763129.719258:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763129.719267:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:3.0:1789763129.719277:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024725632:192.168.203.107@tcp:250:kworker.0 00000008:00100000:0.0:1789763129.719284:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763129.719324:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0380 x1876700024725760/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763129.719340:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763129.719344:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763129.719346:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763129.719351:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2563 00000100:00080000:0.0:1789763129.719357:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763129.719362:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789763129.719367:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763129.719379:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d3100 x1876700024725888/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789763129.719396:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763129.719462:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763129/real 1789763129] req@ffffa06c091d2680 x1876700024725632/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763154 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763129.719484:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d2680 x1876700024725632/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763154 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763129.719499:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763129.719505:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763129.719511:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024725632:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789763129.719546:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024725760:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763129.719570:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763129.719596:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763129/real 1789763129] req@ffffa06c091d0380 x1876700024725760/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763154 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763129.719623:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0380 x1876700024725760/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763154 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763129.719637:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763129.719643:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763129.719649:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024725760:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789763129.719672:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024725888:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763129.719699:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763129.719712:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763129/real 1789763129] req@ffffa06c091d3100 x1876700024725888/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763154 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763129.719724:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d3100 x1876700024725888/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763154 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763129.719732:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763129.719736:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763129.719739:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024725888:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789763129.975071:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763130.230967:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789763130.999066:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763131.254936:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789763132.023019:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763132.278995:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763133.047032:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763133.302942:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789763134.071045:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763134.326959:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789763134.838920:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763134.838927:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763134.838930:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763134.838939:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2568 00000100:00080000:0.0:1789763134.838947:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763134.838953:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789763134.838982:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d1f80 x1876700024726016/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763134.839006:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763134.839020:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763134.839022:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763134.839025:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2568 00000100:00080000:0.0:1789763134.839028:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763134.839031:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789763134.839039:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763134.839054:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0a80 x1876700024726144/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763134.839063:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024726016:192.168.203.107@tcp:250:kworker.0 00000100:00080000:0.0:1789763134.839072:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763134.839077:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763134.839080:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763134.839085:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2568 00000100:00080000:0.0:1789763134.839089:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763134.839094:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789763134.839099:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763134.839109:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0700 x1876700024726272/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789763134.839121:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763134.839157:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763134/real 1789763134] req@ffffa06c091d1f80 x1876700024726016/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763159 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763134.839171:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d1f80 x1876700024726016/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763159 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763134.839178:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763134.839188:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763134.839191:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024726016:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789763134.839210:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024726144:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763134.839227:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763134.839236:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763134/real 1789763134] req@ffffa06c091d0a80 x1876700024726144/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763159 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763134.839244:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0a80 x1876700024726144/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763159 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763134.839264:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763134.839266:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763134.839269:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024726144:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789763134.839273:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024726272:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763134.839288:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763134.839296:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763134/real 1789763134] req@ffffa06c091d0700 x1876700024726272/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763159 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763134.839304:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0700 x1876700024726272/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763159 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763134.839309:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763134.839311:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763134.839313:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024726272:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789763135.094942:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763135.350934:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763136.119046:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763136.375020:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763137.143007:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763137.398971:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789763138.167055:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763138.422951:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763139.191169:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763139.447031:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:3.0:1789763139.959025:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763139.959043:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763139.959052:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763139.959069:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2573 00000100:00080000:3.0:1789763139.959088:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763139.959102:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:3.0:1789763139.959165:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872c700 x1876700024726400/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763139.959236:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763139.959246:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763139.959251:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763139.959261:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2573 00000100:00080000:3.0:1789763139.959270:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763139.959280:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789763139.959300:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763139.959327:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872f100 x1876700024726528/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763139.959353:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1789763139.959383:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024726400:192.168.203.107@tcp:250:kworker.0 00000100:00080000:3.0:1789763139.959383:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763139.959397:0:36:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763139.959417:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2573 00000100:00080000:3.0:1789763139.959425:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763139.959433:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789763139.959447:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1789763139.959510:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789763139.959585:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763139/real 1789763139] req@ffffa06c1872c700 x1876700024726400/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763164 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763139.959642:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872c700 x1876700024726400/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763164 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763139.959658:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763139.959664:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789763139.959670:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024726400:192.168.203.107@tcp:250:kworker.0 00000100:00100000:2.0:1789763139.959722:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024726528:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789763139.959765:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789763139.959784:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763139/real 1789763139] req@ffffa06c1872f100 x1876700024726528/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763164 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763139.959802:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872f100 x1876700024726528/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763164 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763139.959827:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1789763139.959829:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872ca80 x1876700024726656/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763139.959924:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789763139.959936:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024726528:192.168.203.107@tcp:38:kworker.0 00000100:00100000:2.0:1789763139.960055:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024726656:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789763139.960090:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789763139.960107:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763139/real 1789763139] req@ffffa06c1872ca80 x1876700024726656/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763164 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763139.960123:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872ca80 x1876700024726656/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763164 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763139.960151:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763139.960156:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789763139.960161:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024726656:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789763140.215044:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763140.471007:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789763141.239139:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763141.494989:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763142.263095:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763142.518994:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789763143.287093:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763143.542967:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789763144.311120:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763144.567028:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789763145.079003:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763145.079017:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763145.079022:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763145.079036:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2578 00000100:00080000:0.0:1789763145.079050:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763145.079060:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789763145.079117:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0700 x1876700024726784/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763145.079172:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763145.079194:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763145.079196:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763145.079202:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2578 00000100:00080000:0.0:1789763145.079208:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763145.079214:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789763145.079227:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763145.079245:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024726784:192.168.203.107@tcp:250:kworker.0 00000100:00100000:0.0:1789763145.079246:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0a80 x1876700024726912/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763145.079281:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763145.079287:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763145.079289:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763145.079297:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2578 00000100:00080000:0.0:1789763145.079301:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763145.079307:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789763145.079313:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763145.079326:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d1f80 x1876700024727040/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789763145.079350:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789763145.079409:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763145/real 1789763145] req@ffffa06c091d0700 x1876700024726784/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763170 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763145.079438:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0700 x1876700024726784/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763170 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763145.079456:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763145.079462:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789763145.079467:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024726784:192.168.203.107@tcp:250:kworker.0 00000100:00100000:2.0:1789763145.079496:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024726912:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789763145.079523:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789763145.079535:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763145/real 1789763145] req@ffffa06c091d0a80 x1876700024726912/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763170 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763145.079548:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0a80 x1876700024726912/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763170 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763145.079578:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763145.079581:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789763145.079585:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024726912:192.168.203.107@tcp:38:kworker.0 00000100:00100000:2.0:1789763145.079591:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024727040:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789763145.079612:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789763145.079622:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763145/real 1789763145] req@ffffa06c091d1f80 x1876700024727040/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763170 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763145.079633:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d1f80 x1876700024727040/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763170 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763145.079640:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763145.079643:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789763145.079646:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024727040:192.168.203.107@tcp:38:kworker.0 00010000:00010000:2.0:1789763145.334979:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763145.590936:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789763146.359141:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763146.614977:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789763147.383044:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763147.638967:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763148.407085:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763148.662981:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789763149.431058:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763149.686960:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:1.0:1789763150.199010:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763150.199023:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789763150.199028:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763150.199041:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2584 00000100:00080000:1.0:1789763150.199054:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763150.199063:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789763150.199120:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2e300 x1876700024727168/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763150.199200:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763150.199206:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763150.199214:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763150.199220:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2584 00000100:00080000:1.0:1789763150.199226:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763150.199231:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789763150.199244:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763150.199265:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2ed80 x1876700024727296/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763150.199293:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024727168:192.168.203.107@tcp:250:kworker.0 00000100:00080000:1.0:1789763150.199324:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763150.199329:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763150.199333:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763150.199340:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2584 00000100:00080000:1.0:1789763150.199347:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763150.199354:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789763150.199363:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763150.199382:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2c000 x1876700024727424/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763150.199472:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024727296:192.168.203.107@tcp:38:kworker.0 00000100:00100000:2.0:1789763150.199521:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024727424:192.168.203.107@tcp:38:kworker.0 00010000:00010000:1.0:1789763150.455095:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763150.710949:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763151.479214:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763151.734963:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763152.503078:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763152.758995:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000400:00000100:0.0:1789763153.271210:0:2329:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.107:988 00000400:00000100:0.0:1789763153.271251:0:2329:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.107@tcp at host 192.168.203.107 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1789763153.271269:0:2329:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.107@tcp 00000800:00000100:0.0:1789763153.271307:0:2329:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.107@tcp 00000800:00000100:0.0:1789763153.271353:0:2329:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.107@tcp 00000100:00100000:2.0:1789763153.271461:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763150/real 1789763153] req@ffffa06c08e2ed80 x1876700024727296/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763175 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763153.271533:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2ed80 x1876700024727296/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763175 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763153.271549:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763153.271555:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789763153.271560:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024727296:192.168.203.107@tcp:38:kworker.0 00000100:00100000:2.0:1789763153.271580:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763150/real 1789763153] req@ffffa06c08e2c000 x1876700024727424/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763175 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763153.271596:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2c000 x1876700024727424/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763175 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763153.271606:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763153.271611:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789763153.271615:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024727424:192.168.203.107@tcp:38:kworker.0 00000100:00100000:2.0:1789763153.271624:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763150/real 1789763153] req@ffffa06c08e2e300 x1876700024727168/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763175 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763153.271678:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2e300 x1876700024727168/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763175 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763153.271690:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763153.271695:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789763153.271700:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024727168:192.168.203.107@tcp:250:kworker.0 00010000:00010000:3.0:1789763153.527091:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763153.782997:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763154.551073:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763154.806967:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:1.0:1789763155.318977:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763155.318997:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789763155.319005:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763155.319027:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2589 00000100:00080000:1.0:1789763155.319068:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763155.319120:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789763155.319192:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2d500 x1876700024727552/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763155.319251:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763155.319257:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763155.319260:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763155.319267:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2589 00000100:00080000:1.0:1789763155.319276:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763155.319305:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789763155.319325:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763155.319351:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024727552:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789763155.319362:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2e300 x1876700024727680/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763155.319406:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763155.319414:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763155.319419:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763155.319427:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2589 00000100:00080000:1.0:1789763155.319437:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763155.319447:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789763155.319459:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763155.319487:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2c000 x1876700024727808/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789763155.319511:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789763155.319636:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763155/real 1789763155] req@ffffa06c08e2d500 x1876700024727552/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763180 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763155.319681:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2d500 x1876700024727552/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763180 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763155.319704:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763155.319724:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789763155.319732:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024727552:192.168.203.107@tcp:250:kworker.0 00000100:00100000:2.0:1789763155.319777:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024727680:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789763155.319827:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789763155.319906:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763155/real 1789763155] req@ffffa06c08e2e300 x1876700024727680/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763180 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763155.319934:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2e300 x1876700024727680/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763180 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763155.319952:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763155.319961:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789763155.319971:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024727680:192.168.203.107@tcp:38:kworker.0 00000100:00100000:2.0:1789763155.319987:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024727808:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789763155.320048:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789763155.320072:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763155/real 1789763155] req@ffffa06c08e2c000 x1876700024727808/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763180 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763155.320148:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2c000 x1876700024727808/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763180 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763155.320164:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763155.320171:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789763155.320177:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024727808:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789763155.575061:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763155.830932:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763156.599041:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763156.855005:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763157.622987:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763157.878922:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763158.647071:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789763158.903010:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763159.671015:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789763159.926970:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:3.0:1789763160.439020:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763160.439035:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763160.439042:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763160.439057:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2594 00000100:00080000:3.0:1789763160.439098:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763160.439110:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:3.0:1789763160.439164:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872d180 x1876700024727936/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763160.439217:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763160.439224:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763160.439228:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763160.439236:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2594 00000100:00080000:3.0:1789763160.439243:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763160.439250:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789763160.439265:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763160.439290:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872e300 x1876700024728064/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763160.439347:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763160.439358:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:2.0:1789763160.439360:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024727936:192.168.203.107@tcp:250:kworker.0 00000100:00080000:3.0:1789763160.439365:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763160.439380:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2594 00000100:00080000:3.0:1789763160.439402:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763160.439413:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789763160.439426:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763160.439451:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872ce00 x1876700024728192/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789763160.439520:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789763160.439603:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763160/real 1789763160] req@ffffa06c1872d180 x1876700024727936/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763185 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763160.439631:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872d180 x1876700024727936/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763185 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763160.439648:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763160.439654:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789763160.439660:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024727936:192.168.203.107@tcp:250:kworker.0 00000100:00100000:2.0:1789763160.439704:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024728064:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789763160.439756:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789763160.439784:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763160/real 1789763160] req@ffffa06c1872e300 x1876700024728064/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763185 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763160.439812:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872e300 x1876700024728064/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763185 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763160.439947:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763160.439953:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789763160.439958:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024728064:192.168.203.107@tcp:38:kworker.0 00000100:00100000:2.0:1789763160.439967:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024728192:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789763160.440004:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789763160.440021:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763160/real 1789763160] req@ffffa06c1872ce00 x1876700024728192/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763185 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763160.440036:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872ce00 x1876700024728192/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763185 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763160.440047:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763160.440051:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789763160.440055:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024728192:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789763160.695034:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789763160.950931:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763161.719107:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789763161.974934:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763162.743010:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789763162.998967:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763163.767025:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789763164.022983:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763164.791072:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789763165.046957:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789763165.559006:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763165.559017:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763165.559042:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763165.559057:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2599 00000100:00080000:2.0:1789763165.559087:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763165.559094:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789763165.559133:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50000 x1876700024728320/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763165.559165:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763165.559189:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763165.559193:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763165.559199:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2599 00000100:00080000:2.0:1789763165.559206:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1789763165.559210:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024728320:192.168.203.107@tcp:250:kworker.0 00000100:00080000:2.0:1789763165.559211:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789763165.559224:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763165.559242:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51500 x1876700024728448/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763165.559258:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763165.559262:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763165.559265:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763165.559269:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2599 00000100:00080000:2.0:1789763165.559273:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763165.559278:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789763165.559284:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1789763165.559285:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789763165.559297:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50e00 x1876700024728576/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763165.559335:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763165/real 1789763165] req@ffffa06c07e50000 x1876700024728320/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763190 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763165.559360:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50000 x1876700024728320/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763190 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763165.559375:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763165.559379:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763165.559383:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024728320:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789763165.559419:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024728448:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763165.559445:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763165.559458:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763165/real 1789763165] req@ffffa06c07e51500 x1876700024728448/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763190 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763165.559469:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51500 x1876700024728448/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763190 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763165.559478:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763165.559481:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763165.559484:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024728448:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789763165.559489:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024728576:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763165.559505:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763165.559514:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763165/real 1789763165] req@ffffa06c07e50e00 x1876700024728576/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763190 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763165.559525:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50e00 x1876700024728576/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763190 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763165.559537:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763165.559542:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763165.559547:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024728576:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789763165.815019:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789763166.070911:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789763166.839038:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763167.094977:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763167.863085:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763168.118975:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763168.887061:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763169.142968:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763169.911109:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763170.166962:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789763170.679011:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763170.679025:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763170.679030:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763170.679044:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2604 00000100:00080000:2.0:1789763170.679060:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763170.679072:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789763170.679147:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51f80 x1876700024728704/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763170.679190:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763170.679198:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763170.679202:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763170.679210:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2604 00000100:00080000:2.0:1789763170.679220:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763170.679229:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789763170.679246:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763170.679280:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024728704:192.168.203.107@tcp:250:kworker.0 00000100:00100000:2.0:1789763170.679281:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e53b80 x1876700024728832/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763170.679316:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763170.679322:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763170.679325:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763170.679330:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2604 00000100:00080000:2.0:1789763170.679336:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763170.679342:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789763170.679349:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763170.679367:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50380 x1876700024728960/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789763170.679392:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763170.679452:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763170/real 1789763170] req@ffffa06c07e51f80 x1876700024728704/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763195 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763170.679476:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51f80 x1876700024728704/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763195 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763170.679490:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763170.679495:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763170.679501:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024728704:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789763170.679530:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e53b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024728832:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763170.679556:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763170.679571:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763170/real 1789763170] req@ffffa06c07e53b80 x1876700024728832/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763195 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763170.679586:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e53b80 x1876700024728832/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763195 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763170.679597:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763170.679602:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763170.679606:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e53b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024728832:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789763170.679613:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024728960:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763170.679652:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763170.679665:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763170/real 1789763170] req@ffffa06c07e50380 x1876700024728960/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763195 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763170.679680:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50380 x1876700024728960/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763195 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763170.679690:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763170.679693:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763170.679697:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024728960:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789763170.935052:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763171.190946:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763171.959099:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763172.214960:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763172.983038:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763173.238929:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763174.007040:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763174.262987:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763175.031106:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763175.286998:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789763175.799094:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763175.799105:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763175.799111:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763175.799127:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2609 00000100:00080000:0.0:1789763175.799142:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763175.799152:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789763175.799200:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d3100 x1876700024729088/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763175.799237:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763175.799243:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763175.799245:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763175.799250:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2609 00000100:00080000:0.0:1789763175.799256:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763175.799262:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789763175.799276:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763175.799300:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0380 x1876700024729216/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763175.799316:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024729088:192.168.203.107@tcp:250:kworker.0 00000100:00080000:0.0:1789763175.799353:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763175.799360:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763175.799364:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763175.799373:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2609 00000100:00080000:0.0:1789763175.799381:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763175.799389:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789763175.799396:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763175.799414:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d2680 x1876700024729344/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789763175.799721:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763175.799895:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763175/real 1789763175] req@ffffa06c091d3100 x1876700024729088/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763200 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763175.799930:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d3100 x1876700024729088/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763200 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763175.799971:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763175.799991:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763175.800000:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024729088:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789763175.800042:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024729216:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763175.800080:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763175.800099:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763175/real 1789763175] req@ffffa06c091d0380 x1876700024729216/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763200 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763175.800115:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0380 x1876700024729216/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763200 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763175.800127:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763175.800131:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763175.800135:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024729216:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789763175.800142:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024729344:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763175.800163:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763175.800174:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763175/real 1789763175] req@ffffa06c091d2680 x1876700024729344/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763200 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763175.800188:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d2680 x1876700024729344/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763200 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763175.800198:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763175.800201:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763175.800205:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024729344:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789763176.055059:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763176.310991:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763177.079009:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763177.334933:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763178.103033:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763178.358962:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763179.127055:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763179.382976:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763180.151147:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763180.407001:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789763180.918995:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763180.919008:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763180.919014:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763180.919027:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2614 00000100:00080000:0.0:1789763180.919040:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763180.919048:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789763180.919095:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d2a00 x1876700024729472/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763180.919133:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763180.919138:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763180.919141:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763180.919146:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2614 00000100:00080000:0.0:1789763180.919150:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763180.919154:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789763180.919186:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763180.919205:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0e00 x1876700024729600/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763180.919218:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024729472:192.168.203.107@tcp:250:kworker.0 00000100:00080000:0.0:1789763180.919228:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763180.919235:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763180.919238:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763180.919244:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2614 00000100:00080000:0.0:1789763180.919251:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763180.919256:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789763180.919262:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763180.919278:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d1500 x1876700024729728/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789763180.919324:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763180.919383:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763180/real 1789763180] req@ffffa06c091d2a00 x1876700024729472/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763205 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763180.919406:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d2a00 x1876700024729472/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763205 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763180.919421:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763180.919426:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763180.919431:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024729472:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789763180.919458:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024729600:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763180.919484:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763180.919499:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763180/real 1789763180] req@ffffa06c091d0e00 x1876700024729600/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763205 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763180.919514:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0e00 x1876700024729600/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763205 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763180.919524:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763180.919540:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763180.919545:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024729600:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789763180.919553:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024729728:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763180.919579:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763180.919593:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763180/real 1789763180] req@ffffa06c091d1500 x1876700024729728/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763205 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763180.919607:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d1500 x1876700024729728/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763205 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763180.919617:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763180.919621:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763180.919625:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024729728:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789763181.175053:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763181.431009:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763182.199034:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763182.454978:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763183.223109:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763183.479035:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763184.247126:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763184.502984:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763185.271041:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763185.526960:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789763186.039048:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763186.039061:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763186.039067:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763186.039104:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2619 00000100:00080000:2.0:1789763186.039117:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763186.039127:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789763186.039170:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e52680 x1876700024729856/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763186.039208:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763186.039213:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763186.039216:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763186.039221:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2619 00000100:00080000:2.0:1789763186.039225:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763186.039230:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789763186.039242:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763186.039262:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e52680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024729856:192.168.203.107@tcp:250:kworker.0 00000100:00100000:2.0:1789763186.039263:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e53800 x1876700024729984/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763186.039292:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763186.039296:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763186.039298:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763186.039303:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2619 00000100:00080000:2.0:1789763186.039307:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763186.039312:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789763186.039318:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763186.039332:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51c00 x1876700024730112/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789763186.039378:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763186.039446:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763186/real 1789763186] req@ffffa06c07e52680 x1876700024729856/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763211 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763186.039494:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e52680 x1876700024729856/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763211 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763186.039508:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763186.039514:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763186.039519:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e52680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024729856:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789763186.039549:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e53800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024729984:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763186.039582:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763186.039599:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763186/real 1789763186] req@ffffa06c07e53800 x1876700024729984/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763211 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763186.039614:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e53800 x1876700024729984/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763211 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763186.039625:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763186.039629:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763186.039633:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e53800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024729984:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789763186.039640:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024730112:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763186.039659:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763186.039670:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763186/real 1789763186] req@ffffa06c07e51c00 x1876700024730112/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763211 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763186.039684:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51c00 x1876700024730112/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763211 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763186.039748:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763186.039754:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763186.039759:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024730112:192.168.203.107@tcp:38:kworker.0 00010000:00010000:2.0:1789763186.295067:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763186.550994:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763187.319127:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763187.575004:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763188.343121:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763188.598965:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789763189.367123:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763189.622980:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763190.391092:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763190.646946:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:3.0:1789763191.159032:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763191.159045:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763191.159050:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763191.159064:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2624 00000100:00080000:3.0:1789763191.159078:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763191.159087:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:3.0:1789763191.159134:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872c380 x1876700024730240/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763191.159185:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763191.159190:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763191.159193:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763191.159198:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2624 00000100:00080000:3.0:1789763191.159203:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763191.159209:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789763191.159243:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763191.159248:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024730240:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789763191.159267:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872d880 x1876700024730368/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763191.159289:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763191.159293:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763191.159297:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763191.159302:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2624 00000100:00080000:3.0:1789763191.159322:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763191.159328:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789763191.159336:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1789763191.159343:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763191.159362:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872df80 x1876700024730496/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763191.159405:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763191/real 1789763191] req@ffffa06c1872c380 x1876700024730240/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763216 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763191.159431:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872c380 x1876700024730240/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763216 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763191.159461:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763191.159466:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789763191.159472:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024730240:192.168.203.107@tcp:250:kworker.0 00000100:00100000:2.0:1789763191.159503:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024730368:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789763191.159531:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789763191.159546:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763191/real 1789763191] req@ffffa06c1872d880 x1876700024730368/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763216 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763191.159562:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872d880 x1876700024730368/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763216 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763191.159572:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763191.159577:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789763191.159581:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024730368:192.168.203.107@tcp:38:kworker.0 00000100:00100000:2.0:1789763191.159587:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024730496:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789763191.159628:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789763191.159646:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763191/real 1789763191] req@ffffa06c1872df80 x1876700024730496/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763216 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763191.159671:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872df80 x1876700024730496/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763216 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763191.159687:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763191.159693:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789763191.159700:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024730496:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789763191.415031:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763191.670972:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789763192.439046:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763192.694952:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789763193.463030:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763193.718962:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789763194.487090:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763194.743008:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789763195.510988:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763195.767009:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789763196.278999:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763196.279012:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763196.279017:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763196.279030:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2630 00000100:00080000:2.0:1789763196.279043:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763196.279052:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789763196.279100:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e53480 x1876700024730624/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763196.279143:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763196.279148:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763196.279150:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763196.279156:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2630 00000100:00080000:2.0:1789763196.279161:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763196.279167:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:3.0:1789763196.279200:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e53480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024730624:192.168.203.107@tcp:250:kworker.0 00000008:00100000:2.0:1789763196.279202:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763196.279259:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50a80 x1876700024730752/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763196.279275:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763196.279300:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763196.279305:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763196.279313:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2630 00000100:00080000:2.0:1789763196.279335:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763196.279348:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000800:00000100:3.0:1789763196.279356:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000008:00100000:2.0:1789763196.279364:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763196.279396:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50700 x1876700024730880/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763196.279426:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763196/real 1789763196] req@ffffa06c07e53480 x1876700024730624/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763221 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763196.279457:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e53480 x1876700024730624/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763221 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763196.279491:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763196.279497:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763196.279502:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e53480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024730624:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789763196.279547:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024730752:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763196.279582:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763196.279599:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763196/real 1789763196] req@ffffa06c07e50a80 x1876700024730752/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763221 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763196.279615:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50a80 x1876700024730752/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763221 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763196.279626:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763196.279630:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763196.279634:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024730752:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789763196.279642:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024730880:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763196.279662:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763196.279673:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763196/real 1789763196] req@ffffa06c07e50700 x1876700024730880/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763221 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763196.279687:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50700 x1876700024730880/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763221 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763196.279697:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763196.279700:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763196.279704:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024730880:192.168.203.107@tcp:38:kworker.0 00010000:00010000:0.0:1789763196.535082:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763196.790988:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789763197.559071:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763197.814989:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789763198.583086:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763198.838974:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789763199.607067:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763199.862955:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763200.631009:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763200.886982:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789763201.399051:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763201.399064:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763201.399069:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763201.399082:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2635 00000100:00080000:0.0:1789763201.399096:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763201.399105:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789763201.399154:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0000 x1876700024731008/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763201.399225:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763201.399231:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763201.399233:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763201.399239:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2635 00000100:00080000:0.0:1789763201.399244:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763201.399252:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789763201.399264:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763201.399285:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d3b80 x1876700024731136/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763201.399310:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763201.399322:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763201.399332:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1789763201.399334:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024731008:192.168.203.107@tcp:250:kworker.0 00000100:00080000:0.0:1789763201.399350:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2635 00000100:00080000:0.0:1789763201.399363:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763201.399371:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789763201.399384:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763201.399411:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d2300 x1876700024731264/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789763201.399509:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763201.399603:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763201/real 1789763201] req@ffffa06c091d0000 x1876700024731008/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763226 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763201.399634:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0000 x1876700024731008/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763226 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763201.399651:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763201.399658:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763201.399664:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024731008:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789763201.399714:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024731136:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763201.399791:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763201.399816:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763201/real 1789763201] req@ffffa06c091d3b80 x1876700024731136/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763226 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763201.399833:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d3b80 x1876700024731136/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763226 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763201.399898:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763201.399904:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763201.399931:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024731136:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789763201.399942:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024731264:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763201.399978:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763201.399996:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763201/real 1789763201] req@ffffa06c091d2300 x1876700024731264/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763226 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763201.400012:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d2300 x1876700024731264/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763226 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763201.400023:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763201.400027:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763201.400032:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024731264:192.168.203.107@tcp:38:kworker.0 00010000:00010000:2.0:1789763201.655196:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763201.910986:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763202.679001:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763202.934933:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789763203.703030:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763203.958966:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763204.727058:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763204.982981:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763205.751056:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763206.006971:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:1.0:1789763206.519086:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763206.519107:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789763206.519116:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763206.519137:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2640 00000100:00080000:1.0:1789763206.519159:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763206.519171:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789763206.519239:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2ed80 x1876700024731392/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763206.519286:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763206.519318:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763206.519322:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763206.519332:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2640 00000100:00080000:1.0:1789763206.519341:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1789763206.519350:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024731392:192.168.203.107@tcp:250:kworker.0 00000100:00080000:1.0:1789763206.519361:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789763206.519387:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763206.519426:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2d180 x1876700024731520/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763206.519455:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763206.519464:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763206.519470:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:3.0:1789763206.519474:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00080000:1.0:1789763206.519486:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2640 00000100:00080000:1.0:1789763206.519501:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763206.519514:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789763206.519526:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763206.519556:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2ca80 x1876700024731648/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763206.519569:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763206/real 1789763206] req@ffffa06c08e2ed80 x1876700024731392/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763231 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763206.519619:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2ed80 x1876700024731392/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763231 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763206.519664:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763206.519670:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763206.519692:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024731392:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789763206.519760:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024731520:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763206.519800:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763206.519819:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763206/real 1789763206] req@ffffa06c08e2d180 x1876700024731520/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763231 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763206.519835:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2d180 x1876700024731520/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763231 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763206.519890:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763206.519896:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763206.519901:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024731520:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789763206.519909:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024731648:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763206.519936:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763206.519950:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763206/real 1789763206] req@ffffa06c08e2ca80 x1876700024731648/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763231 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763206.519965:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2ca80 x1876700024731648/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763231 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763206.519975:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763206.519979:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763206.519983:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024731648:192.168.203.107@tcp:38:kworker.0 00010000:00010000:1.0:1789763206.775022:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763207.030954:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763207.799068:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763208.054985:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763208.823018:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763209.078941:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789763209.847109:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763210.102953:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789763210.871086:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763211.126957:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789763211.639026:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763211.639044:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763211.639051:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763211.639067:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2645 00000100:00080000:0.0:1789763211.639105:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763211.639120:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789763211.639175:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d2300 x1876700024731776/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763211.639246:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763211.639253:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763211.639257:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763211.639265:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2645 00000100:00080000:0.0:1789763211.639272:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763211.639280:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789763211.639296:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763211.639329:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024731776:192.168.203.107@tcp:250:kworker.0 00000100:00100000:0.0:1789763211.639330:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d3b80 x1876700024731904/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763211.639365:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763211.639373:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763211.639386:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763211.639395:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2645 00000100:00080000:0.0:1789763211.639403:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763211.639412:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789763211.639423:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1789763211.639438:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789763211.639450:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0000 x1876700024732032/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763211.639502:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763211/real 1789763211] req@ffffa06c091d2300 x1876700024731776/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763236 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763211.639534:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d2300 x1876700024731776/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763236 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763211.639549:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763211.639556:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763211.639561:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024731776:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789763211.639594:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024731904:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763211.639628:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763211.639645:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763211/real 1789763211] req@ffffa06c091d3b80 x1876700024731904/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763236 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763211.639664:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d3b80 x1876700024731904/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763236 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763211.639677:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763211.639681:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763211.639729:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024731904:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789763211.639749:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024732032:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763211.639790:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763211.639818:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763211/real 1789763211] req@ffffa06c091d0000 x1876700024732032/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763236 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763211.639835:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0000 x1876700024732032/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763236 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763211.640012:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763211.640028:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763211.640033:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024732032:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789763211.895004:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763212.150964:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789763212.919073:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763213.174958:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789763213.943000:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763214.198926:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789763214.967059:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763215.222977:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789763215.991071:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763216.246950:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:1.0:1789763216.759101:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763216.759115:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789763216.759138:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763216.759157:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2650 00000100:00080000:1.0:1789763216.759173:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763216.759188:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789763216.759257:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2c700 x1876700024732160/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763216.759313:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763216.759321:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763216.759325:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763216.759334:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2650 00000100:00080000:1.0:1789763216.759341:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763216.759349:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789763216.759363:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763216.759376:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024732160:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789763216.759388:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2ca80 x1876700024732288/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763216.759408:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763216.759413:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763216.759417:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763216.759421:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2650 00000100:00080000:1.0:1789763216.759426:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763216.759431:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789763216.759437:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763216.759452:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2d180 x1876700024732416/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763216.759540:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024732288:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789763216.759607:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024732416:192.168.203.107@tcp:38:kworker.0 00010000:00010000:1.0:1789763217.015053:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763217.270985:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763218.039058:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763218.294977:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789763219.063111:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763219.318995:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000400:00000100:2.0:1789763219.831196:0:2330:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.107:988 00000400:00000100:2.0:1789763219.831246:0:2330:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.107@tcp at host 192.168.203.107 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1789763219.831265:0:2330:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.107@tcp 00000800:00000100:2.0:1789763219.831314:0:2330:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.107@tcp 00000800:00000100:2.0:1789763219.831330:0:2330:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.107@tcp 00000100:00100000:3.0:1789763219.831366:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763216/real 1789763219] req@ffffa06c08e2ca80 x1876700024732288/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763241 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763219.831408:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2ca80 x1876700024732288/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763241 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763219.831430:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763219.831438:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763219.831446:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024732288:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789763219.831459:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763216/real 1789763219] req@ffffa06c08e2d180 x1876700024732416/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763241 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763219.831484:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2d180 x1876700024732416/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763241 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763219.831499:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763219.831506:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763219.831512:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024732416:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789763219.831525:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763216/real 1789763219] req@ffffa06c08e2c700 x1876700024732160/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763241 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763219.831549:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2c700 x1876700024732160/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763241 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763219.831563:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763219.831570:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763219.831576:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024732160:192.168.203.107@tcp:250:kworker.0 00010000:00010000:1.0:1789763220.087002:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763220.342983:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763221.111094:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763221.367001:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789763221.878944:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763221.878955:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763221.878960:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763221.878970:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2655 00000100:00080000:0.0:1789763221.878992:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763221.879001:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789763221.879033:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d1500 x1876700024732544/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763221.879055:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763221.879058:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763221.879059:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763221.879062:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2655 00000100:00080000:0.0:1789763221.879065:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763221.879080:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789763221.879088:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763221.879089:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024732544:192.168.203.107@tcp:250:kworker.0 00000100:00100000:0.0:1789763221.879099:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0e00 x1876700024732672/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763221.879109:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763221.879112:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763221.879113:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763221.879115:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2655 00000100:00080000:0.0:1789763221.879118:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763221.879120:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789763221.879123:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763221.879146:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d2a00 x1876700024732800/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789763221.879161:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763221.879223:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763221/real 1789763221] req@ffffa06c091d1500 x1876700024732544/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763246 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763221.879238:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d1500 x1876700024732544/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763246 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763221.879248:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763221.879252:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763221.879257:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024732544:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789763221.879280:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024732672:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763221.879303:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763221.879313:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763221/real 1789763221] req@ffffa06c091d0e00 x1876700024732672/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763246 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763221.879321:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0e00 x1876700024732672/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763246 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763221.879327:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763221.879330:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763221.879332:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024732672:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789763221.879336:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024732800:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763221.879348:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763221.879363:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763221/real 1789763221] req@ffffa06c091d2a00 x1876700024732800/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763246 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763221.879371:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d2a00 x1876700024732800/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763246 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763221.879377:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763221.879379:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763221.879381:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024732800:192.168.203.107@tcp:38:kworker.0 00010000:00010000:2.0:1789763222.135101:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763222.390962:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789763223.159086:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763223.414960:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763224.183032:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763224.438970:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789763225.207039:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763225.462943:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789763226.231023:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763226.486946:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789763226.998941:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763226.998949:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763226.998967:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763226.998976:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2660 00000100:00080000:0.0:1789763226.998985:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763226.998990:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789763226.999038:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d2680 x1876700024732928/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763226.999060:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763226.999063:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763226.999065:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763226.999068:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2660 00000100:00080000:0.0:1789763226.999070:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763226.999073:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789763226.999080:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763226.999105:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0380 x1876700024733056/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763226.999115:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763226.999118:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763226.999119:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1789763226.999123:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024732928:192.168.203.107@tcp:250:kworker.0 00000100:00080000:0.0:1789763226.999124:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2660 00000100:00080000:0.0:1789763226.999129:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763226.999170:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789763226.999175:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763226.999184:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d3100 x1876700024733184/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789763226.999202:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763226.999257:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763226/real 1789763226] req@ffffa06c091d2680 x1876700024732928/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763251 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763226.999271:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d2680 x1876700024732928/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763251 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763226.999279:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763226.999282:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763226.999285:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024732928:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789763226.999302:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024733056:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763226.999327:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763226.999336:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763226/real 1789763226] req@ffffa06c091d0380 x1876700024733056/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763251 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763226.999344:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0380 x1876700024733056/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763251 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763226.999350:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763226.999352:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763226.999354:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024733056:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789763226.999358:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024733184:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763226.999368:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763226.999378:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763226/real 1789763226] req@ffffa06c091d3100 x1876700024733184/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763251 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763226.999390:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d3100 x1876700024733184/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763251 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763226.999398:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763226.999401:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763226.999405:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024733184:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789763227.254974:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763227.510949:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789763228.279089:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763228.534984:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789763229.303029:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763229.558969:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763230.326963:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763230.582973:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789763231.351096:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763231.606969:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789763232.119093:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763232.119105:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763232.119110:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763232.119122:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2665 00000100:00080000:2.0:1789763232.119147:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763232.119156:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789763232.119216:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e52d80 x1876700024733312/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763232.119266:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763232.119274:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763232.119277:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763232.119285:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2665 00000100:00080000:2.0:1789763232.119293:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763232.119308:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789763232.119335:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763232.119366:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e53100 x1876700024733440/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763232.119385:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763232.119390:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763232.119394:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1789763232.119399:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e52d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024733312:192.168.203.107@tcp:250:kworker.0 00000100:00080000:2.0:1789763232.119404:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2665 00000100:00080000:2.0:1789763232.119419:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763232.119439:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789763232.119475:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1789763232.119503:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789763232.119510:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e52a00 x1876700024733568/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763232.119602:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763232/real 1789763232] req@ffffa06c07e52d80 x1876700024733312/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763257 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763232.119629:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e52d80 x1876700024733312/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763257 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763232.119654:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763232.119660:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763232.119665:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e52d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024733312:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789763232.119694:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e53100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024733440:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763232.119722:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763232.119737:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763232/real 1789763232] req@ffffa06c07e53100 x1876700024733440/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763257 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763232.119752:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e53100 x1876700024733440/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763257 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763232.119763:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763232.119767:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763232.119772:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e53100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024733440:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789763232.119778:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e52a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024733568:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763232.119806:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763232.119817:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763232/real 1789763232] req@ffffa06c07e52a00 x1876700024733568/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763257 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763232.119898:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e52a00 x1876700024733568/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763257 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763232.119910:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763232.119914:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763232.119919:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e52a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024733568:192.168.203.107@tcp:38:kworker.0 00010000:00010000:2.0:1789763232.375069:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763232.630949:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763233.399033:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789763233.654966:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789763234.423093:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789763234.678942:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789763235.447043:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789763235.702977:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789763236.471032:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789763236.726963:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789763237.238991:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763237.239008:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763237.239014:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763237.239028:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2671 00000100:00080000:2.0:1789763237.239042:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763237.239051:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789763237.239100:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e52a00 x1876700024733696/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763237.239136:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763237.239142:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763237.239144:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763237.239149:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2671 00000100:00080000:2.0:1789763237.239154:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763237.239158:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789763237.239170:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763237.239186:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e53100 x1876700024733824/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763237.239203:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763237.239208:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:3.0:1789763237.239216:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e52a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024733696:192.168.203.107@tcp:250:kworker.0 00000100:00080000:2.0:1789763237.239229:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763237.239239:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2671 00000100:00080000:2.0:1789763237.239244:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763237.239250:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789763237.239258:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763237.239277:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e52d80 x1876700024733952/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789763237.239312:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763237.239382:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763237/real 1789763237] req@ffffa06c07e52a00 x1876700024733696/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763262 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763237.239410:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e52a00 x1876700024733696/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763262 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763237.239426:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763237.239433:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763237.239439:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e52a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024733696:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789763237.239468:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e53100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024733824:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763237.239494:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763237.239537:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763237/real 1789763237] req@ffffa06c07e53100 x1876700024733824/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763262 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763237.239553:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e53100 x1876700024733824/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763262 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763237.239577:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763237.239582:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763237.239587:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e53100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024733824:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789763237.239595:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e52d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024733952:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763237.239625:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763237.239639:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763237/real 1789763237] req@ffffa06c07e52d80 x1876700024733952/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763262 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763237.239653:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e52d80 x1876700024733952/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763262 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763237.239662:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763237.239666:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763237.239669:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e52d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024733952:192.168.203.107@tcp:38:kworker.0 00010000:00010000:2.0:1789763237.495125:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789763237.750990:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789763238.519148:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763238.775010:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789763239.542992:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763239.798990:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789763240.567110:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763240.822981:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789763241.591057:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763241.846968:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:1.0:1789763242.359075:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763242.359087:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789763242.359092:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763242.359104:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2676 00000100:00080000:1.0:1789763242.359117:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763242.359127:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789763242.359173:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2ed80 x1876700024734080/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763242.359211:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763242.359216:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763242.359219:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763242.359224:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2676 00000100:00080000:1.0:1789763242.359229:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763242.359233:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789763242.359244:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763242.359260:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2c000 x1876700024734208/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763242.359276:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763242.359281:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:3.0:1789763242.359304:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024734080:192.168.203.107@tcp:250:kworker.0 00000100:00080000:1.0:1789763242.359307:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763242.359324:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2676 00000100:00080000:1.0:1789763242.359329:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763242.359334:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789763242.359344:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763242.359363:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2e300 x1876700024734336/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789763242.359410:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763242.359510:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763242/real 1789763242] req@ffffa06c08e2ed80 x1876700024734080/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763267 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763242.359546:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2ed80 x1876700024734080/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763267 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763242.359581:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763242.359587:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763242.359593:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024734080:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789763242.359603:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024734208:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763242.359639:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763242.359662:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763242/real 1789763242] req@ffffa06c08e2c000 x1876700024734208/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763267 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763242.359688:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2c000 x1876700024734208/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763267 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763242.359705:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763242.359712:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763242.359719:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024734208:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789763242.359761:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024734336:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763242.359795:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763242.359810:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763242/real 1789763242] req@ffffa06c08e2e300 x1876700024734336/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763267 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763242.359826:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2e300 x1876700024734336/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763267 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763242.359836:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763242.359841:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763242.359889:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024734336:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789763242.615071:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763242.870987:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789763243.639034:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763243.894952:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789763244.663063:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763244.918988:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763245.686973:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763245.942957:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789763246.711032:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763246.966958:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789763247.479018:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763247.479035:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763247.479042:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763247.479061:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2681 00000100:00080000:0.0:1789763247.479078:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763247.479093:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789763247.479149:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d1f80 x1876700024734464/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763247.479190:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763247.479196:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763247.479198:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763247.479204:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2681 00000100:00080000:0.0:1789763247.479209:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763247.479215:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789763247.479253:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763247.479274:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0a80 x1876700024734592/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763247.479294:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763247.479298:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763247.479302:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763247.479307:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2681 00000100:00080000:0.0:1789763247.479315:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763247.479320:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789763247.479329:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763247.479356:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0700 x1876700024734720/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763247.479358:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024734464:192.168.203.107@tcp:250:kworker.0 00000800:00000100:3.0:1789763247.480164:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763247.480238:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763247/real 1789763247] req@ffffa06c091d1f80 x1876700024734464/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763272 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763247.480271:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d1f80 x1876700024734464/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763272 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763247.480289:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763247.480296:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763247.480302:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024734464:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789763247.480315:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024734592:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763247.480349:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763247.480365:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763247/real 1789763247] req@ffffa06c091d0a80 x1876700024734592/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763272 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763247.480381:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0a80 x1876700024734592/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763272 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763247.480392:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763247.480396:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763247.480413:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024734592:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789763247.480481:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024734720:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763247.480515:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763247.480531:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763247/real 1789763247] req@ffffa06c091d0700 x1876700024734720/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763272 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763247.480547:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0700 x1876700024734720/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763272 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763247.480557:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763247.480561:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763247.480565:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024734720:192.168.203.107@tcp:38:kworker.0 00010000:00010000:2.0:1789763247.735136:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763247.990973:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763248.759052:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763249.014974:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789763249.783078:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763250.038964:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789763250.807063:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763251.062930:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763251.831071:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763252.086953:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:1.0:1789763252.599017:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763252.599035:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789763252.599042:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763252.599059:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2686 00000100:00080000:1.0:1789763252.599076:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763252.599089:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789763252.599136:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2d500 x1876700024734848/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763252.599178:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763252.599183:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763252.599186:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763252.599191:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2686 00000100:00080000:1.0:1789763252.599196:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763252.599201:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789763252.599213:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763252.599252:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2e300 x1876700024734976/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763252.599271:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024734848:192.168.203.107@tcp:250:kworker.0 00000100:00080000:1.0:1789763252.599272:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763252.599290:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763252.599295:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763252.599300:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2686 00000100:00080000:1.0:1789763252.599306:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763252.599311:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789763252.599319:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763252.599336:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2c000 x1876700024735104/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789763252.599377:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763252.599463:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763252/real 1789763252] req@ffffa06c08e2d500 x1876700024734848/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763277 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763252.599488:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2d500 x1876700024734848/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763277 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763252.599527:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763252.599536:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763252.599544:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024734848:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789763252.599584:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024734976:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763252.599631:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763252.599652:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763252/real 1789763252] req@ffffa06c08e2e300 x1876700024734976/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763277 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763252.599668:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2e300 x1876700024734976/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763277 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763252.599679:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763252.599683:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763252.599689:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024734976:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789763252.599701:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024735104:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763252.599737:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763252.599760:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763252/real 1789763252] req@ffffa06c08e2c000 x1876700024735104/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763277 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763252.599786:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2c000 x1876700024735104/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763277 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763252.599800:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763252.599804:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763252.599809:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024735104:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789763252.855111:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789763253.111062:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789763253.879071:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789763254.134995:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789763254.903037:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789763255.158959:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789763255.926972:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789763256.182958:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789763256.951081:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789763257.207034:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789763257.719054:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763257.719071:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763257.719078:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763257.719092:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2691 00000100:00080000:2.0:1789763257.719106:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763257.719116:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789763257.719167:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50700 x1876700024735232/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763257.719208:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763257.719214:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763257.719236:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763257.719242:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2691 00000100:00080000:2.0:1789763257.719248:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763257.719254:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789763257.719267:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763257.719288:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50a80 x1876700024735360/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763257.719304:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763257.719310:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763257.719312:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763257.719318:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2691 00000100:00080000:2.0:1789763257.719321:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763257.719326:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789763257.719332:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763257.719370:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024735232:192.168.203.107@tcp:250:kworker.0 00000100:00100000:2.0:1789763257.719381:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e53480 x1876700024735488/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789763257.719531:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763257.719606:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763257/real 1789763257] req@ffffa06c07e50700 x1876700024735232/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763282 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763257.719634:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50700 x1876700024735232/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763282 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763257.719651:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763257.719657:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763257.719663:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024735232:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789763257.719674:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024735360:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763257.719703:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763257.719718:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763257/real 1789763257] req@ffffa06c07e50a80 x1876700024735360/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763282 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763257.719733:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50a80 x1876700024735360/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763282 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763257.719744:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763257.719749:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763257.719753:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024735360:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789763257.719790:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e53480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024735488:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763257.719811:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763257.719909:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763257/real 1789763257] req@ffffa06c07e53480 x1876700024735488/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763282 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763257.719926:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e53480 x1876700024735488/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763282 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763257.719937:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763257.719942:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763257.719946:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e53480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024735488:192.168.203.107@tcp:38:kworker.0 00010000:00010000:2.0:1789763257.975112:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789763258.230964:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789763258.999173:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763259.255088:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789763260.023045:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763260.278970:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789763261.047066:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763261.302959:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789763262.071081:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763262.326957:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789763262.838987:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763262.838999:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763262.839005:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763262.839018:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2696 00000100:00080000:2.0:1789763262.839032:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763262.839058:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789763262.839101:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51c00 x1876700024735616/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763262.839137:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763262.839142:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763262.839144:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763262.839150:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2696 00000100:00080000:2.0:1789763262.839155:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763262.839159:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789763262.839171:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763262.839186:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e53800 x1876700024735744/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763262.839201:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763262.839206:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763262.839209:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763262.839216:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2696 00000100:00080000:2.0:1789763262.839221:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763262.839227:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789763262.839233:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763262.839245:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e52680 x1876700024735872/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763262.839261:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024735616:192.168.203.107@tcp:250:kworker.0 00000800:00000100:3.0:1789763262.839383:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763262.839437:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763262/real 1789763262] req@ffffa06c07e51c00 x1876700024735616/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763287 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763262.839462:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51c00 x1876700024735616/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763287 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763262.839476:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763262.839481:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763262.839486:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024735616:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789763262.839496:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e53800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024735744:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763262.839538:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763262.839554:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763262/real 1789763262] req@ffffa06c07e53800 x1876700024735744/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763287 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763262.839584:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e53800 x1876700024735744/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763287 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763262.839595:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763262.839599:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763262.839604:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e53800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024735744:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789763262.839633:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e52680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024735872:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763262.839660:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763262.839673:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763262/real 1789763262] req@ffffa06c07e52680 x1876700024735872/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763287 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763262.839688:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e52680 x1876700024735872/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763287 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763262.839697:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763262.839701:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763262.839705:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e52680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024735872:192.168.203.107@tcp:38:kworker.0 00010000:00010000:0.0:1789763263.095105:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763263.350966:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763264.119012:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763264.374975:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789763265.143057:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763265.398940:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789763266.167073:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763266.422977:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763267.191038:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763267.446965:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789763267.958997:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763267.959010:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763267.959015:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763267.959029:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2701 00000100:00080000:0.0:1789763267.959065:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763267.959075:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789763267.959122:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0700 x1876700024736000/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763267.959162:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763267.959167:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763267.959170:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763267.959175:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2701 00000100:00080000:0.0:1789763267.959180:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763267.959184:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789763267.959195:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763267.959210:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0a80 x1876700024736128/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763267.959224:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763267.959228:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763267.959231:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763267.959235:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2701 00000100:00080000:0.0:1789763267.959240:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763267.959246:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789763267.959254:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763267.959268:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d1f80 x1876700024736256/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763267.959333:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024736000:192.168.203.107@tcp:250:kworker.0 00000800:00000100:3.0:1789763267.959477:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763267.959546:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763267/real 1789763267] req@ffffa06c091d0700 x1876700024736000/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763292 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763267.959574:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0700 x1876700024736000/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763292 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763267.959591:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763267.959597:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763267.959603:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024736000:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789763267.959614:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024736128:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763267.959661:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763267.959678:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763267/real 1789763267] req@ffffa06c091d0a80 x1876700024736128/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763292 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763267.959695:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0a80 x1876700024736128/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763292 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763267.959705:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763267.959710:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763267.959714:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024736128:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789763267.959751:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024736256:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763267.959773:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763267.959786:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763267/real 1789763267] req@ffffa06c091d1f80 x1876700024736256/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763292 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763267.959822:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d1f80 x1876700024736256/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763292 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763267.959833:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763267.959837:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763267.959841:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024736256:192.168.203.107@tcp:38:kworker.0 00010000:00010000:1.0:1789763268.215138:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763268.470967:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763269.239116:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763269.494978:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763270.263098:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763270.518975:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789763271.287047:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763271.542987:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763272.311062:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763272.566956:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789763273.079016:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763273.079043:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763273.079048:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763273.079059:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2706 00000100:00080000:0.0:1789763273.079089:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763273.079098:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789763273.079141:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d3100 x1876700024736384/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763273.079191:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763273.079214:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763273.079217:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763273.079224:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2706 00000100:00080000:0.0:1789763273.079230:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763273.079255:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789763273.079288:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763273.079312:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0380 x1876700024736512/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763273.079330:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763273.079335:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763273.079338:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763273.079344:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2706 00000100:00080000:0.0:1789763273.079350:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763273.079355:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789763273.079361:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763273.079390:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024736384:192.168.203.107@tcp:250:kworker.0 00000100:00100000:0.0:1789763273.079399:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d2680 x1876700024736640/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789763273.079531:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763273.079603:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763273/real 1789763273] req@ffffa06c091d3100 x1876700024736384/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763298 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763273.079631:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d3100 x1876700024736384/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763298 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763273.079649:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763273.079655:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763273.079661:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024736384:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789763273.079672:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024736512:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763273.079739:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763273.079756:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763273/real 1789763273] req@ffffa06c091d0380 x1876700024736512/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763298 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763273.079771:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0380 x1876700024736512/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763298 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763273.079792:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763273.079797:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763273.079801:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024736512:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789763273.079841:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024736640:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763273.079912:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763273.079928:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763273/real 1789763273] req@ffffa06c091d2680 x1876700024736640/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763298 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763273.079943:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d2680 x1876700024736640/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763298 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763273.079954:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763273.079958:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763273.079962:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024736640:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789763273.335140:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763273.591001:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763274.359056:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763274.614998:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763275.383040:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763275.638943:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763276.407060:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763276.662951:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763277.431012:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763277.686991:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789763278.198977:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763278.198990:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763278.198996:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763278.199008:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2712 00000100:00080000:0.0:1789763278.199022:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763278.199031:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789763278.199080:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d2a00 x1876700024736768/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763278.199122:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763278.199128:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763278.199130:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763278.199135:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2712 00000100:00080000:0.0:1789763278.199140:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763278.199144:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789763278.199155:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763278.199174:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0e00 x1876700024736896/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763278.199192:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763278.199218:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763278.199220:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763278.199225:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2712 00000100:00080000:0.0:1789763278.199229:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763278.199245:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789763278.199273:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763278.199290:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d1500 x1876700024737024/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763278.199302:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024736768:192.168.203.107@tcp:250:kworker.0 00000800:00000100:3.0:1789763278.199391:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763278.199448:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763278/real 1789763278] req@ffffa06c091d2a00 x1876700024736768/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763303 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763278.199472:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d2a00 x1876700024736768/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763303 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763278.199486:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763278.199491:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763278.199496:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024736768:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789763278.199505:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024736896:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763278.199529:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763278.199542:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763278/real 1789763278] req@ffffa06c091d0e00 x1876700024736896/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763303 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763278.199557:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0e00 x1876700024736896/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763303 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763278.199567:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763278.199571:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763278.199575:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024736896:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789763278.199604:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024737024:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763278.199642:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763278.199656:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763278/real 1789763278] req@ffffa06c091d1500 x1876700024737024/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763303 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763278.199671:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d1500 x1876700024737024/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763303 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763278.199681:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763278.199685:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763278.199689:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024737024:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789763278.455054:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763278.710965:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763279.479056:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763279.734954:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763280.502977:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763280.758968:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763281.526972:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789763281.782983:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763282.550977:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789763282.807007:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789763283.319929:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763283.319938:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763283.319941:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763283.319976:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2717 00000100:00080000:0.0:1789763283.319989:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763283.319998:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789763283.320041:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0000 x1876700024737152/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763283.320077:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763283.320088:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763283.320091:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763283.320096:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2717 00000100:00080000:0.0:1789763283.320101:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763283.320106:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789763283.320118:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763283.320146:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d3b80 x1876700024737280/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763283.320184:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024737152:192.168.203.107@tcp:250:kworker.0 00000100:00080000:0.0:1789763283.320202:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763283.320209:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763283.320225:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763283.320234:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2717 00000100:00080000:0.0:1789763283.320242:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763283.320249:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789763283.320258:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763283.320280:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d2300 x1876700024737408/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763283.320332:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024737280:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789763283.320400:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024737408:192.168.203.107@tcp:38:kworker.0 00010000:00010000:2.0:1789763283.574978:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789763283.831010:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763284.598967:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789763284.854918:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763285.622952:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789763285.878991:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000400:00000100:2.0:1789763286.391240:0:2331:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.107:988 00000400:00000100:2.0:1789763286.391308:0:2331:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.107@tcp at host 192.168.203.107 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1789763286.391334:0:2331:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.107@tcp 00000800:00000100:2.0:1789763286.391404:0:2331:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.107@tcp 00000800:00000100:2.0:1789763286.391423:0:2331:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.107@tcp 00000100:00100000:3.0:1789763286.391478:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763283/real 1789763286] req@ffffa06c091d3b80 x1876700024737280/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763308 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763286.391564:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d3b80 x1876700024737280/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763308 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763286.391587:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763286.391596:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763286.391605:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024737280:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789763286.391651:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763283/real 1789763286] req@ffffa06c091d2300 x1876700024737408/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763308 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763286.391677:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d2300 x1876700024737408/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763308 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763286.391693:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763286.391700:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763286.391707:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024737408:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789763286.391905:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763283/real 1789763286] req@ffffa06c091d0000 x1876700024737152/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763308 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763286.391938:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0000 x1876700024737152/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763308 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763286.391954:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763286.391961:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763286.391969:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024737152:192.168.203.107@tcp:250:kworker.0 00010000:00010000:2.0:1789763286.647064:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789763286.903024:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763287.670967:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789763287.926976:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789763288.439080:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763288.439088:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763288.439091:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763288.439099:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2722 00000100:00080000:2.0:1789763288.439107:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763288.439113:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789763288.439152:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50380 x1876700024737536/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763288.439176:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763288.439179:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763288.439180:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763288.439183:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2722 00000100:00080000:2.0:1789763288.439186:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763288.439188:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789763288.439195:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763288.439204:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e53b80 x1876700024737664/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763288.439217:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024737536:192.168.203.107@tcp:250:kworker.0 00000100:00080000:2.0:1789763288.439225:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763288.439247:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763288.439249:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763288.439254:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2722 00000100:00080000:2.0:1789763288.439260:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763288.439265:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789763288.439271:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763288.439286:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51f80 x1876700024737792/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789763288.439301:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763288.439355:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763288/real 1789763288] req@ffffa06c07e50380 x1876700024737536/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763313 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763288.439373:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50380 x1876700024737536/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763313 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763288.439384:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763288.439388:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763288.439392:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024737536:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789763288.439416:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e53b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024737664:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763288.439453:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763288.439468:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763288/real 1789763288] req@ffffa06c07e53b80 x1876700024737664/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763313 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763288.439481:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e53b80 x1876700024737664/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763313 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763288.439489:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763288.439493:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763288.439497:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e53b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024737664:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789763288.439518:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024737792:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763288.439543:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763288.439650:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763288/real 1789763288] req@ffffa06c07e51f80 x1876700024737792/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763313 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763288.439687:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51f80 x1876700024737792/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763313 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763288.439696:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763288.439700:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763288.439704:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024737792:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789763288.694949:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789763288.951001:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789763289.719002:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763289.975024:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789763290.742987:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763290.998936:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763291.766964:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763292.022932:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789763292.790991:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763293.046922:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789763293.558920:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763293.558928:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763293.558932:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763293.558940:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2727 00000100:00080000:2.0:1789763293.558949:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763293.558955:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789763293.558987:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50e00 x1876700024737920/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763293.559010:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763293.559013:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763293.559015:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763293.559017:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2727 00000100:00080000:2.0:1789763293.559020:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763293.559026:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789763293.559048:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763293.559058:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51500 x1876700024738048/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763293.559068:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763293.559071:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763293.559073:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1789763293.559073:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024737920:192.168.203.107@tcp:250:kworker.0 00000100:00080000:2.0:1789763293.559079:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2727 00000100:00080000:2.0:1789763293.559099:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763293.559103:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789763293.559107:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763293.559117:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50000 x1876700024738176/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789763293.559153:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763293.559193:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763293/real 1789763293] req@ffffa06c07e50e00 x1876700024737920/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763318 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763293.559212:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50e00 x1876700024737920/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763318 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763293.559224:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763293.559227:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763293.559232:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024737920:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789763293.559253:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024738048:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763293.559273:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763293.559293:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763293/real 1789763293] req@ffffa06c07e51500 x1876700024738048/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763318 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763293.559306:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51500 x1876700024738048/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763318 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763293.559315:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763293.559318:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763293.559322:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024738048:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789763293.559329:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024738176:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763293.559347:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763293.559357:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763293/real 1789763293] req@ffffa06c07e50000 x1876700024738176/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763318 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763293.559370:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50000 x1876700024738176/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763318 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763293.559378:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763293.559381:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763293.559385:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024738176:192.168.203.107@tcp:38:kworker.0 00010000:00010000:1.0:1789763293.814961:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763294.070904:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763294.838963:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763295.094900:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789763295.862961:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763296.118900:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789763296.886953:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763297.142897:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763297.910948:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763298.166936:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789763298.678933:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763298.678942:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763298.678946:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763298.678956:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2732 00000100:00080000:2.0:1789763298.678966:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763298.678973:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789763298.679018:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51180 x1876700024738304/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763298.679061:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763298.679065:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763298.679082:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763298.679106:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2732 00000100:00080000:2.0:1789763298.679113:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763298.679119:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:3.0:1789763298.679126:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024738304:192.168.203.107@tcp:250:kworker.0 00000008:00100000:2.0:1789763298.679134:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763298.679154:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e52300 x1876700024738432/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763298.679169:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763298.679173:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763298.679176:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763298.679180:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2732 00000100:00080000:2.0:1789763298.679196:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763298.679201:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789763298.679224:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1789763298.679232:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789763298.679247:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51880 x1876700024738560/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763298.679301:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763298/real 1789763298] req@ffffa06c07e51180 x1876700024738304/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763323 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763298.679326:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51180 x1876700024738304/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763323 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763298.679340:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763298.679345:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763298.679351:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024738304:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789763298.679381:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e52300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024738432:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763298.679423:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763298.679433:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763298/real 1789763298] req@ffffa06c07e52300 x1876700024738432/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763323 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763298.679443:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e52300 x1876700024738432/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763323 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763298.679450:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763298.679453:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763298.679456:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e52300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024738432:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789763298.679460:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024738560:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763298.679472:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763298.679479:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763298/real 1789763298] req@ffffa06c07e51880 x1876700024738560/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763323 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763298.679502:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51880 x1876700024738560/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763323 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763298.679509:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763298.679511:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763298.679514:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024738560:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789763298.934964:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763299.190947:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763299.958948:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763300.214937:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763300.982940:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763301.238918:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763302.006947:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763302.262937:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763303.030937:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763303.286909:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789763303.798961:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763303.798972:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763303.798976:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763303.798986:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2737 00000100:00080000:0.0:1789763303.798998:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763303.799016:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789763303.799054:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0000 x1876700024738688/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763303.799080:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763303.799085:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763303.799087:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763303.799092:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2737 00000100:00080000:0.0:1789763303.799095:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763303.799111:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789763303.799121:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763303.799132:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024738688:192.168.203.107@tcp:250:kworker.0 00000100:00100000:0.0:1789763303.799139:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d2300 x1876700024738816/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763303.799155:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763303.799159:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763303.799162:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763303.799166:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2737 00000100:00080000:0.0:1789763303.799170:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763303.799174:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789763303.799180:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763303.799191:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d3b80 x1876700024738944/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789763303.799201:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763303.799253:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763303/real 1789763303] req@ffffa06c091d0000 x1876700024738688/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763328 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763303.799272:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0000 x1876700024738688/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763328 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763303.799282:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763303.799287:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763303.799291:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024738688:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789763303.799318:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024738816:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763303.799340:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763303.799352:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763303/real 1789763303] req@ffffa06c091d2300 x1876700024738816/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763328 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763303.799365:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d2300 x1876700024738816/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763328 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763303.799374:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763303.799378:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763303.799381:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024738816:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789763303.799388:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024738944:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763303.799407:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763303.799418:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763303/real 1789763303] req@ffffa06c091d3b80 x1876700024738944/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763328 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763303.799430:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d3b80 x1876700024738944/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763328 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763303.799438:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763303.799442:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763303.799446:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024738944:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789763304.054952:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763304.310918:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763305.078914:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763305.334924:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763306.102959:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763306.358905:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763307.126946:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763307.382909:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763308.150935:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763308.406913:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789763308.918928:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763308.918938:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763308.918942:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763308.918951:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2742 00000100:00080000:0.0:1789763308.918961:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763308.918978:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789763308.919012:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d1500 x1876700024739072/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763308.919041:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763308.919045:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763308.919060:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763308.919065:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2742 00000100:00080000:0.0:1789763308.919070:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763308.919078:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:3.0:1789763308.919078:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024739072:192.168.203.107@tcp:250:kworker.0 00000008:00100000:0.0:1789763308.919090:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763308.919108:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0e00 x1876700024739200/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763308.919124:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763308.919128:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763308.919130:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:3.0:1789763308.919136:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00080000:0.0:1789763308.919137:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2742 00000100:00080000:0.0:1789763308.919144:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763308.919149:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789763308.919155:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763308.919168:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d2a00 x1876700024739328/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763308.919179:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763308/real 1789763308] req@ffffa06c091d1500 x1876700024739072/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763333 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763308.919194:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d1500 x1876700024739072/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763333 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763308.919203:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763308.919206:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763308.919211:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024739072:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789763308.919230:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024739200:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763308.919244:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763308.919253:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763308/real 1789763308] req@ffffa06c091d0e00 x1876700024739200/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763333 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763308.919261:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0e00 x1876700024739200/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763333 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763308.919266:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763308.919268:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763308.919270:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024739200:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789763308.919281:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024739328:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763308.919295:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763308.919302:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763308/real 1789763308] req@ffffa06c091d2a00 x1876700024739328/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763333 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763308.919310:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d2a00 x1876700024739328/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763333 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763308.919315:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763308.919317:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763308.919319:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024739328:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789763309.174948:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763309.431017:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763310.198987:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763310.454993:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763311.223001:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789763311.479037:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789763312.247073:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789763312.502976:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763313.271073:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789763313.526962:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:1.0:1789763314.039070:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763314.039083:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789763314.039088:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763314.039101:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2747 00000100:00080000:1.0:1789763314.039115:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763314.039124:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789763314.039176:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2ed80 x1876700024739456/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763314.039237:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763314.039261:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763314.039264:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763314.039270:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2747 00000100:00080000:1.0:1789763314.039277:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763314.039284:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789763314.039298:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763314.039322:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2c700 x1876700024739584/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763314.039326:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024739456:192.168.203.107@tcp:250:kworker.0 00000100:00080000:1.0:1789763314.039361:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763314.039384:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763314.039388:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763314.039397:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2747 00000100:00080000:1.0:1789763314.039407:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763314.039417:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789763314.039429:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1789763314.039440:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789763314.039460:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2d180 x1876700024739712/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763314.039508:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763314/real 1789763314] req@ffffa06c08e2ed80 x1876700024739456/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763339 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763314.039536:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2ed80 x1876700024739456/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763339 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763314.039551:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763314.039556:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763314.039584:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024739456:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789763314.039616:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024739584:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763314.039649:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763314.039676:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763314/real 1789763314] req@ffffa06c08e2c700 x1876700024739584/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763339 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763314.039692:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2c700 x1876700024739584/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763339 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763314.039702:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763314.039706:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763314.039711:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024739584:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789763314.039717:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024739712:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763314.039736:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763314.039748:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763314/real 1789763314] req@ffffa06c08e2d180 x1876700024739712/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763339 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763314.039762:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2d180 x1876700024739712/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763339 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763314.039772:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763314.039775:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763314.039779:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024739712:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789763314.295018:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789763314.551074:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789763315.319155:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789763315.574993:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789763316.343056:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789763316.598997:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789763317.367074:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789763317.622983:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789763318.391044:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789763318.646965:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:1.0:1789763319.159011:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763319.159029:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789763319.159037:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763319.159054:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2752 00000100:00080000:1.0:1789763319.159072:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763319.159086:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789763319.159153:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2ca80 x1876700024739840/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763319.159225:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763319.159231:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763319.159235:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763319.159242:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2752 00000100:00080000:1.0:1789763319.159250:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763319.159258:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789763319.159269:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024739840:192.168.203.107@tcp:250:kworker.0 00000008:00100000:1.0:1789763319.159276:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763319.159306:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2d180 x1876700024739968/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763319.159328:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763319.159333:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763319.159336:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763319.159342:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2752 00000100:00080000:1.0:1789763319.159348:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763319.159353:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789763319.159360:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1789763319.159363:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789763319.159390:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2c700 x1876700024740096/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789763319.159446:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763319/real 1789763319] req@ffffa06c08e2ca80 x1876700024739840/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763344 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763319.159485:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2ca80 x1876700024739840/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763344 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763319.159507:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763319.159513:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789763319.159521:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024739840:192.168.203.107@tcp:250:kworker.0 00000100:00100000:0.0:1789763319.159555:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024739968:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789763319.159589:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789763319.159605:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763319/real 1789763319] req@ffffa06c08e2d180 x1876700024739968/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763344 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763319.159621:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2d180 x1876700024739968/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763344 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763319.159633:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763319.159637:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789763319.159642:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024739968:192.168.203.107@tcp:38:kworker.0 00000100:00100000:0.0:1789763319.159649:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024740096:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789763319.159673:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789763319.159684:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763319/real 1789763319] req@ffffa06c08e2c700 x1876700024740096/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763344 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763319.159724:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2c700 x1876700024740096/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763344 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763319.159736:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763319.159740:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789763319.159745:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024740096:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789763319.415030:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789763319.670998:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789763320.439152:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763320.695060:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789763321.463053:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763321.718953:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763322.487055:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763322.742983:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763323.511119:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763323.766976:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:1.0:1789763324.279018:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763324.279034:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789763324.279041:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763324.279123:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2758 00000100:00080000:1.0:1789763324.279145:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763324.279162:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789763324.279243:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2ed80 x1876700024740224/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763324.279294:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763324.279300:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763324.279303:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763324.279310:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2758 00000100:00080000:1.0:1789763324.279316:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763324.279323:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789763324.279339:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763324.279362:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2c000 x1876700024740352/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789763324.279373:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024740224:192.168.203.107@tcp:250:kworker.0 00000100:00080000:1.0:1789763324.279387:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763324.279393:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763324.279395:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763324.279400:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2758 00000100:00080000:1.0:1789763324.279404:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763324.279409:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789763324.279416:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763324.279431:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2e300 x1876700024740480/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789763324.279522:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789763324.279598:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763324/real 1789763324] req@ffffa06c08e2ed80 x1876700024740224/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763349 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763324.279623:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2ed80 x1876700024740224/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763349 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763324.279638:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763324.279643:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789763324.279649:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024740224:192.168.203.107@tcp:250:kworker.0 00000100:00100000:0.0:1789763324.279684:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024740352:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789763324.279711:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789763324.279726:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763324/real 1789763324] req@ffffa06c08e2c000 x1876700024740352/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763349 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763324.279740:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2c000 x1876700024740352/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763349 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763324.279766:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763324.279770:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789763324.279774:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024740352:192.168.203.107@tcp:38:kworker.0 00000100:00100000:0.0:1789763324.279782:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024740480:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789763324.279808:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789763324.279819:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763324/real 1789763324] req@ffffa06c08e2e300 x1876700024740480/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763349 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763324.279832:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2e300 x1876700024740480/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763349 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763324.279842:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763324.279899:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789763324.279908:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024740480:192.168.203.107@tcp:38:kworker.0 00010000:00010000:0.0:1789763324.535046:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763324.790957:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763325.559014:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763325.815038:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763326.583032:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763326.838985:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789763327.607123:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763327.862998:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789763328.630942:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763328.886976:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:1.0:1789763329.398910:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763329.398918:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789763329.398921:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763329.398929:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2763 00000100:00080000:1.0:1789763329.398938:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763329.398944:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789763329.398975:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2d500 x1876700024740608/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763329.398996:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763329.398998:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763329.399000:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763329.399002:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2763 00000100:00080000:1.0:1789763329.399005:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763329.399007:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789763329.399014:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763329.399024:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2e300 x1876700024740736/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763329.399033:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763329.399036:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763329.399037:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763329.399039:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2763 00000100:00080000:1.0:1789763329.399041:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1789763329.399045:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024740608:192.168.203.107@tcp:250:kworker.0 00000100:00080000:1.0:1789763329.399057:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789763329.399082:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763329.399100:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2c000 x1876700024740864/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789763329.399147:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789763329.399199:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763329/real 1789763329] req@ffffa06c08e2d500 x1876700024740608/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763354 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763329.399222:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2d500 x1876700024740608/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763354 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763329.399236:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763329.399241:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789763329.399259:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024740608:192.168.203.107@tcp:250:kworker.0 00000100:00100000:0.0:1789763329.399269:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024740736:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789763329.399321:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789763329.399338:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763329/real 1789763329] req@ffffa06c08e2e300 x1876700024740736/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763354 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763329.399354:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2e300 x1876700024740736/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763354 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763329.399364:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763329.399369:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789763329.399373:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024740736:192.168.203.107@tcp:38:kworker.0 00000100:00100000:0.0:1789763329.399402:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024740864:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789763329.399423:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789763329.399435:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763329/real 1789763329] req@ffffa06c08e2c000 x1876700024740864/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763354 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763329.399449:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2c000 x1876700024740864/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763354 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763329.399459:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763329.399462:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789763329.399466:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024740864:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789763329.654998:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763329.911016:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789763330.679099:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763330.935000:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789763331.703032:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763331.958964:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789763332.726955:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763332.982986:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789763333.751158:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763334.006973:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789763334.519003:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763334.519022:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763334.519030:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763334.519049:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2768 00000100:00080000:0.0:1789763334.519069:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763334.519084:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789763334.519154:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d2680 x1876700024740992/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763334.519208:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763334.519214:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763334.519217:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763334.519223:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2768 00000100:00080000:0.0:1789763334.519228:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763334.519234:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789763334.519248:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763334.519265:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0380 x1876700024741120/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763334.519280:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763334.519285:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763334.519288:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763334.519292:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2768 00000100:00080000:0.0:1789763334.519296:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763334.519301:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789763334.519307:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763334.519348:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d3100 x1876700024741248/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763334.519370:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024740992:192.168.203.107@tcp:250:kworker.0 00000800:00000100:3.0:1789763334.519497:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763334.519566:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763334/real 1789763334] req@ffffa06c091d2680 x1876700024740992/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763359 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763334.519593:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d2680 x1876700024740992/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763359 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763334.519608:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763334.519614:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763334.519620:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024740992:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789763334.519646:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024741120:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763334.519679:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763334.519696:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763334/real 1789763334] req@ffffa06c091d0380 x1876700024741120/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763359 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763334.519712:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0380 x1876700024741120/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763359 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763334.519723:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763334.519728:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763334.519732:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024741120:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789763334.519764:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024741248:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763334.519821:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763334.519838:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763334/real 1789763334] req@ffffa06c091d3100 x1876700024741248/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763359 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763334.519912:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d3100 x1876700024741248/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763359 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763334.519923:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763334.519927:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763334.519931:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024741248:192.168.203.107@tcp:38:kworker.0 00010000:00010000:2.0:1789763334.775024:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763335.030919:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789763335.799187:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763336.054932:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789763336.823053:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763337.078900:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789763337.847026:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763338.102900:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789763338.871004:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763339.126964:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789763339.639007:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763339.639023:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763339.639031:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763339.639048:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2773 00000100:00080000:0.0:1789763339.639066:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763339.639078:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789763339.639150:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d1f80 x1876700024741376/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763339.639210:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763339.639219:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763339.639223:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763339.639232:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2773 00000100:00080000:0.0:1789763339.639241:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763339.639249:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789763339.639266:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763339.639290:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0a80 x1876700024741504/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763339.639313:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024741376:192.168.203.107@tcp:250:kworker.0 00000100:00080000:0.0:1789763339.639315:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763339.639335:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763339.639365:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763339.639378:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2773 00000100:00080000:0.0:1789763339.639385:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763339.639393:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789763339.639403:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763339.639420:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0700 x1876700024741632/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789763339.639454:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763339.639528:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763339/real 1789763339] req@ffffa06c091d1f80 x1876700024741376/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763364 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763339.639566:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d1f80 x1876700024741376/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763364 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763339.639589:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763339.639599:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763339.639607:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024741376:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789763339.639643:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024741504:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763339.639679:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763339.639698:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763339/real 1789763339] req@ffffa06c091d0a80 x1876700024741504/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763364 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763339.639714:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0a80 x1876700024741504/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763364 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763339.639725:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763339.639730:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763339.639734:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024741504:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789763339.639771:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024741632:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763339.639817:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763339.639899:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763339/real 1789763339] req@ffffa06c091d0700 x1876700024741632/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763364 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763339.639957:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0700 x1876700024741632/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763364 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763339.639972:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763339.639976:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763339.639981:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024741632:192.168.203.107@tcp:38:kworker.0 00010000:00010000:1.0:1789763339.895044:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763340.150956:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789763340.919101:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763341.174951:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789763341.943118:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763342.198968:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789763342.966998:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763343.222988:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789763343.991047:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763344.246960:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789763344.758981:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763344.758999:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763344.759006:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763344.759024:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2778 00000100:00080000:0.0:1789763344.759042:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763344.759056:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789763344.759117:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0700 x1876700024741760/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763344.759165:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763344.759173:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763344.759177:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763344.759185:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2778 00000100:00080000:0.0:1789763344.759193:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763344.759199:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789763344.759211:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763344.759251:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0a80 x1876700024741888/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763344.759268:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763344.759273:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763344.759276:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763344.759281:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2778 00000100:00080000:0.0:1789763344.759286:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1789763344.759294:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024741760:192.168.203.107@tcp:250:kworker.0 00000100:00080000:0.0:1789763344.759299:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789763344.759316:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763344.759332:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d1f80 x1876700024742016/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789763344.759395:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763344.759467:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763344/real 1789763344] req@ffffa06c091d0700 x1876700024741760/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763369 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763344.759494:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0700 x1876700024741760/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763369 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763344.759508:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763344.759514:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763344.759519:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024741760:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789763344.759528:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024741888:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763344.759554:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763344.759569:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763344/real 1789763344] req@ffffa06c091d0a80 x1876700024741888/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763369 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763344.759602:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0a80 x1876700024741888/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763369 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763344.759613:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763344.759617:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763344.759621:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024741888:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789763344.759650:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024742016:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763344.759676:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763344.759689:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763344/real 1789763344] req@ffffa06c091d1f80 x1876700024742016/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763369 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763344.759731:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d1f80 x1876700024742016/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763369 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763344.759742:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763344.759746:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763344.759751:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024742016:192.168.203.107@tcp:38:kworker.0 00010000:00010000:0.0:1789763345.015106:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763345.270935:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789763346.039055:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763346.294937:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789763347.063015:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763347.318907:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789763348.087098:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763348.342912:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789763349.111106:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763349.366970:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:1.0:1789763349.878996:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763349.879010:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789763349.879015:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763349.879027:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2783 00000100:00080000:1.0:1789763349.879040:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763349.879048:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789763349.879115:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2ed80 x1876700024742144/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763349.879157:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763349.879162:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763349.879164:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763349.879170:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2783 00000100:00080000:1.0:1789763349.879184:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763349.879193:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789763349.879209:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763349.879232:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024742144:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789763349.879246:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2c700 x1876700024742272/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763349.879320:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763349.879325:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763349.879327:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763349.879334:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2783 00000100:00080000:1.0:1789763349.879342:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763349.879349:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789763349.879358:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763349.879380:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2d180 x1876700024742400/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763349.879389:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024742272:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789763349.879475:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024742400:192.168.203.107@tcp:38:kworker.0 00010000:00010000:1.0:1789763350.134993:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763350.390914:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789763351.158993:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763351.414956:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789763352.183135:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763352.439015:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000400:00000100:0.0:1789763352.951253:0:2328:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.107:988 00000400:00000100:0.0:1789763352.951320:0:2328:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.107@tcp at host 192.168.203.107 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1789763352.951345:0:2328:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.107@tcp 00000800:00000100:0.0:1789763352.951401:0:2328:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.107@tcp 00000800:00000100:0.0:1789763352.951412:0:2328:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.107@tcp 00000100:00100000:3.0:1789763352.951527:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763349/real 1789763352] req@ffffa06c08e2d180 x1876700024742400/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763374 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763352.951575:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2d180 x1876700024742400/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763374 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763352.951597:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763352.951607:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763352.951616:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024742400:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789763352.951633:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763349/real 1789763352] req@ffffa06c08e2c700 x1876700024742272/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763374 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763352.951690:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2c700 x1876700024742272/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763374 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763352.951711:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763352.951718:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763352.951726:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024742272:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789763352.951740:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763349/real 1789763352] req@ffffa06c08e2ed80 x1876700024742144/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763374 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763352.951767:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2ed80 x1876700024742144/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763374 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763352.951800:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763352.951808:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763352.951816:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024742144:192.168.203.107@tcp:250:kworker.0 00010000:00010000:2.0:1789763353.207087:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763353.462951:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789763354.230994:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763354.486930:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:1.0:1789763354.998982:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763354.998995:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789763354.999001:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763354.999016:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2788 00000100:00080000:1.0:1789763354.999034:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763354.999046:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789763354.999102:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2ca80 x1876700024742528/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763354.999161:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763354.999166:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763354.999168:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763354.999172:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2788 00000100:00080000:1.0:1789763354.999178:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763354.999195:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789763354.999206:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763354.999227:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2ed80 x1876700024742656/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763354.999228:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024742528:192.168.203.107@tcp:250:kworker.0 00000100:00080000:1.0:1789763354.999250:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763354.999255:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763354.999257:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763354.999261:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2788 00000100:00080000:1.0:1789763354.999267:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763354.999273:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789763354.999280:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763354.999296:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2c700 x1876700024742784/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789763354.999301:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789763354.999346:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763354/real 1789763354] req@ffffa06c08e2ca80 x1876700024742528/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763379 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763354.999372:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2ca80 x1876700024742528/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763379 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763354.999383:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763354.999387:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789763354.999392:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024742528:192.168.203.107@tcp:250:kworker.0 00000100:00100000:2.0:1789763354.999416:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024742656:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789763354.999446:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789763354.999463:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763354/real 1789763354] req@ffffa06c08e2ed80 x1876700024742656/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763379 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763354.999493:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2ed80 x1876700024742656/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763379 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763354.999503:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763354.999508:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789763354.999519:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024742656:192.168.203.107@tcp:38:kworker.0 00000100:00100000:2.0:1789763354.999530:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024742784:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789763354.999553:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789763354.999574:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763354/real 1789763354] req@ffffa06c08e2c700 x1876700024742784/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763379 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763354.999586:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2c700 x1876700024742784/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763379 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763354.999595:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763354.999598:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789763354.999602:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024742784:192.168.203.107@tcp:38:kworker.0 00010000:00010000:2.0:1789763355.254936:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763355.510916:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763356.279008:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763356.534949:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789763357.303069:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763357.558959:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789763358.327126:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763358.582950:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763359.351055:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763359.606946:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:1.0:1789763360.118920:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763360.118931:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789763360.118935:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763360.118946:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2793 00000100:00080000:1.0:1789763360.118976:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763360.118983:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789763360.119026:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2d180 x1876700024742912/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763360.119050:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763360.119054:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763360.119076:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763360.119080:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2793 00000100:00080000:1.0:1789763360.119086:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763360.119090:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789763360.119099:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763360.119113:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2c000 x1876700024743040/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763360.119118:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024742912:192.168.203.107@tcp:250:kworker.0 00000100:00080000:1.0:1789763360.119133:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763360.119137:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763360.119138:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763360.119141:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2793 00000100:00080000:1.0:1789763360.119145:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763360.119150:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789763360.119154:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763360.119162:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2e300 x1876700024743168/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789763360.119199:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789763360.119235:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763360/real 1789763360] req@ffffa06c08e2d180 x1876700024742912/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763385 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763360.119250:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2d180 x1876700024742912/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763385 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763360.119259:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763360.119263:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789763360.119266:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024742912:192.168.203.107@tcp:250:kworker.0 00000100:00100000:2.0:1789763360.119286:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024743040:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789763360.119300:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789763360.119309:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763360/real 1789763360] req@ffffa06c08e2c000 x1876700024743040/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763385 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763360.119336:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2c000 x1876700024743040/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763385 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763360.119343:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763360.119345:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789763360.119348:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024743040:192.168.203.107@tcp:38:kworker.0 00000100:00100000:2.0:1789763360.119352:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024743168:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789763360.119366:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789763360.119373:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763360/real 1789763360] req@ffffa06c08e2e300 x1876700024743168/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763385 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763360.119381:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2e300 x1876700024743168/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763385 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763360.119386:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763360.119388:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789763360.119390:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024743168:192.168.203.107@tcp:38:kworker.0 00010000:00010000:2.0:1789763360.375012:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763360.630990:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763361.399043:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763361.654962:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763362.423113:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763362.679008:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763363.447040:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763363.702980:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763364.470999:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763364.726923:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789763365.238980:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763365.238988:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763365.238991:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763365.238999:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2799 00000100:00080000:2.0:1789763365.239008:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763365.239013:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789763365.239059:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51880 x1876700024743296/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763365.239083:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763365.239086:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763365.239087:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763365.239090:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2799 00000100:00080000:2.0:1789763365.239093:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763365.239095:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789763365.239103:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763365.239111:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e52300 x1876700024743424/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763365.239118:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763365.239120:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763365.239122:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763365.239123:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2799 00000100:00080000:2.0:1789763365.239125:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763365.239127:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789763365.239129:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763365.239135:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51180 x1876700024743552/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763365.239166:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024743296:192.168.203.107@tcp:250:kworker.0 00000800:00000100:3.0:1789763365.239221:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763365.239254:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763365/real 1789763365] req@ffffa06c07e51880 x1876700024743296/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763390 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763365.239267:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51880 x1876700024743296/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763390 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763365.239284:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763365.239287:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763365.239290:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024743296:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789763365.239296:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e52300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024743424:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763365.239313:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763365.239321:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763365/real 1789763365] req@ffffa06c07e52300 x1876700024743424/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763390 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763365.239329:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e52300 x1876700024743424/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763390 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763365.239334:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763365.239336:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763365.239338:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e52300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024743424:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789763365.239342:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024743552:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763365.239351:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763365.239357:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763365/real 1789763365] req@ffffa06c07e51180 x1876700024743552/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763390 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763365.239364:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51180 x1876700024743552/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763390 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763365.239369:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763365.239371:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763365.239373:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024743552:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789763365.494961:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763365.750977:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763366.518961:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763366.774914:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789763367.542983:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763367.798979:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763368.567057:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763368.822969:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763369.590970:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763369.846945:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789763370.359038:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763370.359056:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763370.359063:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763370.359107:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2804 00000100:00080000:2.0:1789763370.359123:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763370.359133:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789763370.359179:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50000 x1876700024743680/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763370.359223:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763370.359228:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763370.359230:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763370.359236:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2804 00000100:00080000:2.0:1789763370.359240:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763370.359244:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789763370.359256:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763370.359274:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51500 x1876700024743808/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763370.359292:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763370.359296:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763370.359300:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1789763370.359309:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024743680:192.168.203.107@tcp:250:kworker.0 00000100:00080000:2.0:1789763370.359311:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2804 00000100:00080000:2.0:1789763370.359323:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763370.359330:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789763370.359336:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763370.359353:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50e00 x1876700024743936/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789763370.359409:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763370.359466:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763370/real 1789763370] req@ffffa06c07e50000 x1876700024743680/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763395 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763370.359489:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50000 x1876700024743680/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763395 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763370.359503:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763370.359508:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763370.359514:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024743680:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789763370.359571:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024743808:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763370.359603:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763370.359619:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763370/real 1789763370] req@ffffa06c07e51500 x1876700024743808/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763395 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763370.359635:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51500 x1876700024743808/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763395 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763370.359667:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763370.359671:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763370.359676:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024743808:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789763370.359684:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024743936:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763370.359712:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763370.359725:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763370/real 1789763370] req@ffffa06c07e50e00 x1876700024743936/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763395 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763370.359739:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50e00 x1876700024743936/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763395 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763370.359749:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763370.359752:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763370.359756:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024743936:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789763370.615063:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763370.870980:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763371.638980:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763371.894950:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763372.663023:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763372.918946:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763373.687066:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763373.942947:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789763374.711053:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763374.966974:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789763375.478983:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763375.479000:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763375.479007:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763375.479023:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2809 00000100:00080000:0.0:1789763375.479039:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763375.479077:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789763375.479122:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d3100 x1876700024744064/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763375.479166:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763375.479171:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763375.479182:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763375.479187:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2809 00000100:00080000:0.0:1789763375.479194:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763375.479199:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789763375.479212:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763375.479228:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0380 x1876700024744192/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763375.479247:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763375.479257:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:3.0:1789763375.479257:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024744064:192.168.203.107@tcp:250:kworker.0 00000100:00080000:0.0:1789763375.479266:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763375.479281:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2809 00000100:00080000:0.0:1789763375.479285:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763375.479289:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789763375.479296:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763375.479312:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d2680 x1876700024744320/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789763375.479360:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763375.479427:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763375/real 1789763375] req@ffffa06c091d3100 x1876700024744064/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763400 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763375.479452:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d3100 x1876700024744064/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763400 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763375.479466:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763375.479471:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763375.479477:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024744064:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789763375.479547:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024744192:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763375.479580:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763375.479598:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763375/real 1789763375] req@ffffa06c091d0380 x1876700024744192/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763400 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763375.479614:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0380 x1876700024744192/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763400 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763375.479625:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763375.479629:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763375.479633:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024744192:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789763375.479641:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024744320:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763375.479660:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763375.479671:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763375/real 1789763375] req@ffffa06c091d2680 x1876700024744320/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763400 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763375.479685:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d2680 x1876700024744320/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763400 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763375.479713:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763375.479717:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763375.479721:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024744320:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789763375.735021:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763375.990914:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789763376.759031:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763377.014950:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789763377.783071:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763378.038981:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763378.807120:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763379.062946:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763379.831081:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763380.087012:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789763380.598960:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763380.598984:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763380.599012:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763380.599024:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2814 00000100:00080000:0.0:1789763380.599037:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763380.599046:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789763380.599090:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d2a00 x1876700024744448/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763380.599128:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763380.599141:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763380.599144:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763380.599149:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2814 00000100:00080000:0.0:1789763380.599154:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763380.599159:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789763380.599172:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763380.599188:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0e00 x1876700024744576/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763380.599205:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763380.599210:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763380.599213:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1789763380.599222:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024744448:192.168.203.107@tcp:250:kworker.0 00000100:00080000:0.0:1789763380.599224:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2814 00000100:00080000:0.0:1789763380.599236:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763380.599242:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789763380.599249:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763380.599265:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d1500 x1876700024744704/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789763380.599318:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763380.599398:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763380/real 1789763380] req@ffffa06c091d2a00 x1876700024744448/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763405 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763380.599427:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d2a00 x1876700024744448/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763405 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763380.599467:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763380.599485:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763380.599494:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024744448:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789763380.599532:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024744576:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763380.599574:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763380.599594:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763380/real 1789763380] req@ffffa06c091d0e00 x1876700024744576/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763405 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763380.599624:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0e00 x1876700024744576/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763405 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763380.599635:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763380.599640:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763380.599644:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024744576:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789763380.599652:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024744704:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763380.599679:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763380.599692:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763380/real 1789763380] req@ffffa06c091d1500 x1876700024744704/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763405 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763380.599707:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d1500 x1876700024744704/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763405 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763380.599717:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763380.599720:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763380.599724:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024744704:192.168.203.107@tcp:38:kworker.0 00010000:00010000:2.0:1789763380.855092:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789763381.111029:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763381.879039:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763382.135055:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789763382.903048:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763383.158951:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789763383.927071:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763384.182974:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763384.951150:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763385.206972:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:3.0:1789763385.719072:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763385.719085:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763385.719115:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763385.719128:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2819 00000100:00080000:3.0:1789763385.719141:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763385.719151:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:3.0:1789763385.719193:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872f800 x1876700024744832/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763385.719234:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763385.719239:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763385.719242:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763385.719247:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2819 00000100:00080000:3.0:1789763385.719251:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763385.719255:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789763385.719267:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763385.719284:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872df80 x1876700024744960/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763385.719298:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763385.719303:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763385.719305:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763385.719309:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2819 00000100:00080000:3.0:1789763385.719313:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763385.719318:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789763385.719324:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763385.719336:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872d880 x1876700024745088/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763385.719366:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024744832:192.168.203.107@tcp:250:kworker.0 00000800:00000100:2.0:1789763385.719487:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789763385.719539:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763385/real 1789763385] req@ffffa06c1872f800 x1876700024744832/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763410 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763385.719563:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872f800 x1876700024744832/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763410 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763385.719577:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763385.719583:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789763385.719588:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024744832:192.168.203.107@tcp:250:kworker.0 00000100:00100000:2.0:1789763385.719598:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024744960:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789763385.719622:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789763385.719635:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763385/real 1789763385] req@ffffa06c1872df80 x1876700024744960/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763410 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763385.719650:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872df80 x1876700024744960/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763410 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763385.719660:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763385.719664:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789763385.719682:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024744960:192.168.203.107@tcp:38:kworker.0 00000100:00100000:2.0:1789763385.719713:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024745088:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789763385.719739:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789763385.719752:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763385/real 1789763385] req@ffffa06c1872d880 x1876700024745088/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763410 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763385.719767:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872d880 x1876700024745088/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763410 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763385.719777:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763385.719781:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789763385.719785:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024745088:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789763385.974992:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763386.230950:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789763386.999098:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763387.254983:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763388.023055:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763388.278969:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763389.047028:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763389.302965:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789763390.071081:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763390.326972:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:3.0:1789763390.839049:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763390.839060:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763390.839065:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763390.839079:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2824 00000100:00080000:3.0:1789763390.839095:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763390.839105:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:3.0:1789763390.839153:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872c380 x1876700024745216/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763390.839187:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763390.839193:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763390.839196:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763390.839201:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2824 00000100:00080000:3.0:1789763390.839206:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763390.839210:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789763390.839222:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763390.839241:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872ce00 x1876700024745344/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763390.839268:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024745216:192.168.203.107@tcp:250:kworker.0 00000100:00080000:3.0:1789763390.839301:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763390.839308:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763390.839312:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763390.839320:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2824 00000100:00080000:3.0:1789763390.839328:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763390.839335:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789763390.839344:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763390.839390:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872e300 x1876700024745472/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789763390.839401:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789763390.839478:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763390/real 1789763390] req@ffffa06c1872c380 x1876700024745216/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763415 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763390.839504:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872c380 x1876700024745216/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763415 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763390.839518:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763390.839523:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789763390.839528:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024745216:192.168.203.107@tcp:250:kworker.0 00000100:00100000:2.0:1789763390.839558:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024745344:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789763390.839588:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789763390.839620:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763390/real 1789763390] req@ffffa06c1872ce00 x1876700024745344/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763415 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763390.839636:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872ce00 x1876700024745344/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763415 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763390.839647:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763390.839651:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789763390.839656:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024745344:192.168.203.107@tcp:38:kworker.0 00000100:00100000:2.0:1789763390.839666:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024745472:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789763390.839693:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789763390.839707:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763390/real 1789763390] req@ffffa06c1872e300 x1876700024745472/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763415 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763390.839721:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872e300 x1876700024745472/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763415 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763390.839731:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763390.839734:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789763390.839738:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024745472:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789763391.095007:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763391.350964:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789763392.119117:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763392.374972:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763393.143060:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763393.398992:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763394.167065:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763394.422970:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789763395.191033:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763395.446980:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:1.0:1789763395.959002:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763395.959023:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789763395.959031:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763395.959051:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2829 00000100:00080000:1.0:1789763395.959089:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763395.959127:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789763395.959189:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2d500 x1876700024745600/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763395.959233:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763395.959239:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763395.959241:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763395.959248:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2829 00000100:00080000:1.0:1789763395.959254:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763395.959258:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789763395.959273:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763395.959291:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2e300 x1876700024745728/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763395.959308:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763395.959313:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763395.959316:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763395.959321:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2829 00000100:00080000:1.0:1789763395.959325:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763395.959357:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789763395.959366:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763395.959401:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2c000 x1876700024745856/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763395.959415:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024745600:192.168.203.107@tcp:250:kworker.0 00000800:00000100:2.0:1789763395.959530:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789763395.959607:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763395/real 1789763395] req@ffffa06c08e2d500 x1876700024745600/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763420 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763395.959634:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2d500 x1876700024745600/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763420 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763395.959662:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763395.959668:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789763395.959673:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024745600:192.168.203.107@tcp:250:kworker.0 00000100:00100000:2.0:1789763395.959684:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024745728:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789763395.959716:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789763395.959731:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763395/real 1789763395] req@ffffa06c08e2e300 x1876700024745728/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763420 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763395.959766:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2e300 x1876700024745728/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763420 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763395.959777:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763395.959781:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789763395.959785:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024745728:192.168.203.107@tcp:38:kworker.0 00000100:00100000:2.0:1789763395.959822:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024745856:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789763395.959891:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789763395.959909:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763395/real 1789763395] req@ffffa06c08e2c000 x1876700024745856/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763420 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763395.959924:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2c000 x1876700024745856/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763420 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763395.959935:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763395.959940:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789763395.959944:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024745856:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789763396.215027:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763396.471018:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763397.239119:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763397.494987:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789763398.263070:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763398.518972:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789763399.287078:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763399.542980:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763400.311043:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763400.566970:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:1.0:1789763401.078950:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763401.078962:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789763401.078987:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763401.079000:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2834 00000100:00080000:1.0:1789763401.079013:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763401.079022:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789763401.079065:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2d180 x1876700024745984/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763401.079104:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763401.079109:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763401.079112:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763401.079117:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2834 00000100:00080000:1.0:1789763401.079121:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763401.079125:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789763401.079136:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763401.079152:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2c700 x1876700024746112/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763401.079167:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763401.079173:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763401.079176:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763401.079180:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2834 00000100:00080000:1.0:1789763401.079184:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763401.079188:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789763401.079195:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763401.079205:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2ed80 x1876700024746240/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763401.079244:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024745984:192.168.203.107@tcp:250:kworker.0 00000800:00000100:2.0:1789763401.079405:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789763401.079478:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763401/real 1789763401] req@ffffa06c08e2d180 x1876700024745984/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763426 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763401.079505:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2d180 x1876700024745984/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763426 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763401.079523:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763401.079529:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789763401.079535:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024745984:192.168.203.107@tcp:250:kworker.0 00000100:00100000:2.0:1789763401.079547:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024746112:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789763401.079575:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789763401.079590:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763401/real 1789763401] req@ffffa06c08e2c700 x1876700024746112/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763426 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763401.079605:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2c700 x1876700024746112/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763426 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763401.079616:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763401.079620:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789763401.079624:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024746112:192.168.203.107@tcp:38:kworker.0 00000100:00100000:2.0:1789763401.079662:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024746240:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789763401.079683:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789763401.079722:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763401/real 1789763401] req@ffffa06c08e2ed80 x1876700024746240/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763426 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763401.079737:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2ed80 x1876700024746240/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763426 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763401.079748:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763401.079752:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789763401.079756:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024746240:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789763401.335053:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763401.590960:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789763402.359002:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763402.614915:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763403.383076:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763403.638930:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763404.406983:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789763404.662956:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789763405.431012:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789763405.687015:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789763406.199005:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763406.199018:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763406.199024:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763406.199037:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2840 00000100:00080000:2.0:1789763406.199063:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763406.199073:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789763406.199119:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51f80 x1876700024746368/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763406.199158:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763406.199163:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763406.199166:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763406.199171:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2840 00000100:00080000:2.0:1789763406.199176:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763406.199180:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789763406.199192:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763406.199240:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e53b80 x1876700024746496/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763406.199287:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763406.199292:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763406.199295:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763406.199302:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2840 00000100:00080000:2.0:1789763406.199308:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763406.199315:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789763406.199326:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763406.199345:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50380 x1876700024746624/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763406.199359:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024746368:192.168.203.107@tcp:250:kworker.0 00000800:00000100:3.0:1789763406.199492:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763406.199595:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763406/real 1789763406] req@ffffa06c07e51f80 x1876700024746368/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763431 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763406.199623:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51f80 x1876700024746368/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763431 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763406.199640:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763406.199646:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763406.199651:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024746368:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789763406.199663:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e53b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024746496:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763406.199694:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763406.199709:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763406/real 1789763406] req@ffffa06c07e53b80 x1876700024746496/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763431 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763406.199725:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e53b80 x1876700024746496/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763431 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763406.199735:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763406.199739:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763406.199743:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e53b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024746496:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789763406.199781:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024746624:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763406.199832:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763406.199847:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763406/real 1789763406] req@ffffa06c07e50380 x1876700024746624/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763431 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763406.199862:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50380 x1876700024746624/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763431 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763406.199930:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763406.199935:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763406.199939:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024746624:192.168.203.107@tcp:38:kworker.0 00010000:00010000:1.0:1789763406.455064:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789763406.710963:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763407.478952:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789763407.734923:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763408.502953:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789763408.758957:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789763409.527031:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789763409.782964:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763410.551064:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789763410.806967:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:1.0:1789763411.319013:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763411.319027:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789763411.319032:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763411.319045:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2845 00000100:00080000:1.0:1789763411.319058:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763411.319067:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789763411.319114:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2ca80 x1876700024746752/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763411.319150:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763411.319155:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763411.319158:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763411.319163:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2845 00000100:00080000:1.0:1789763411.319167:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763411.319171:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789763411.319182:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763411.319200:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2ed80 x1876700024746880/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763411.319236:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763411.319243:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763411.319246:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763411.319254:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2845 00000100:00080000:1.0:1789763411.319277:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763411.319287:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:3.0:1789763411.319287:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024746752:192.168.203.107@tcp:250:kworker.0 00000008:00100000:1.0:1789763411.319304:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763411.319324:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2c700 x1876700024747008/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789763411.319415:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763411.319479:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763411/real 1789763411] req@ffffa06c08e2ca80 x1876700024746752/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763436 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763411.319504:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2ca80 x1876700024746752/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763436 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763411.319519:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763411.319524:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763411.319529:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024746752:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789763411.319539:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024746880:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763411.319576:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763411.319607:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763411/real 1789763411] req@ffffa06c08e2ed80 x1876700024746880/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763436 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763411.319629:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2ed80 x1876700024746880/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763436 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763411.319640:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763411.319644:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763411.319649:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024746880:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789763411.319678:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024747008:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763411.319715:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763411.319730:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763411/real 1789763411] req@ffffa06c08e2c700 x1876700024747008/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763436 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763411.319744:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2c700 x1876700024747008/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763436 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763411.319755:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763411.319758:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763411.319762:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024747008:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789763411.574950:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789763411.830986:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763412.599028:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763412.855105:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763413.622999:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763413.879014:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763414.647074:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763414.902998:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763415.671011:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763415.926975:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789763416.439002:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763416.439020:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763416.439027:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763416.439055:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2850 00000100:00080000:0.0:1789763416.439071:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763416.439082:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789763416.439139:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d3b80 x1876700024747136/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763416.439199:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763416.439207:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763416.439210:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763416.439218:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2850 00000100:00080000:0.0:1789763416.439225:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763416.439248:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:3.0:1789763416.439261:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024747136:192.168.203.107@tcp:250:kworker.0 00000008:00100000:0.0:1789763416.439265:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763416.439294:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d2300 x1876700024747264/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763416.439313:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763416.439319:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763416.439322:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763416.439327:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2850 00000100:00080000:0.0:1789763416.439342:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763416.439347:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789763416.439354:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763416.439370:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0000 x1876700024747392/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763416.439464:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024747264:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789763416.439517:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024747392:192.168.203.107@tcp:38:kworker.0 00010000:00010000:1.0:1789763416.695068:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763416.951046:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763417.718989:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763417.974944:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789763418.742993:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763418.998905:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000400:00000100:0.0:1789763419.511321:0:2329:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.107:988 00000400:00000100:0.0:1789763419.511383:0:2329:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.107@tcp at host 192.168.203.107 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1789763419.511406:0:2329:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.107@tcp 00000800:00000100:0.0:1789763419.511463:0:2329:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.107@tcp 00000800:00000100:0.0:1789763419.511475:0:2329:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.107@tcp 00000100:00100000:1.0:1789763419.511523:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763416/real 1789763419] req@ffffa06c091d2300 x1876700024747264/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763441 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763419.511576:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d2300 x1876700024747264/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763441 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763419.511603:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763419.511614:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789763419.511624:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024747264:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789763419.511642:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763416/real 1789763419] req@ffffa06c091d0000 x1876700024747392/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763441 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763419.511672:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0000 x1876700024747392/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763441 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763419.511692:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763419.511701:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789763419.511711:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024747392:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789763419.511731:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763416/real 1789763419] req@ffffa06c091d3b80 x1876700024747136/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763441 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763419.511790:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d3b80 x1876700024747136/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763441 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763419.511806:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763419.511813:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789763419.511820:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024747136:192.168.203.107@tcp:250:kworker.0 00010000:00010000:0.0:1789763419.767129:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763420.022952:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763420.790979:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763421.046931:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789763421.558965:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763421.558979:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763421.558984:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763421.558997:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2855 00000100:00080000:2.0:1789763421.559010:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763421.559019:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789763421.559084:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e52680 x1876700024747520/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763421.559132:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763421.559136:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763421.559151:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763421.559157:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2855 00000100:00080000:2.0:1789763421.559172:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763421.559177:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789763421.559187:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763421.559202:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e52680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024747520:192.168.203.107@tcp:250:kworker.0 00000100:00100000:2.0:1789763421.559210:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e53800 x1876700024747648/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763421.559245:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763421.559250:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763421.559253:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763421.559259:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2855 00000100:00080000:2.0:1789763421.559263:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763421.559267:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789763421.559273:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763421.559291:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51c00 x1876700024747776/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789763421.559305:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789763421.559359:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763421/real 1789763421] req@ffffa06c07e52680 x1876700024747520/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763446 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763421.559386:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e52680 x1876700024747520/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763446 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763421.559402:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763421.559408:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789763421.559414:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e52680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024747520:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789763421.559445:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e53800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024747648:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789763421.559495:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789763421.559517:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763421/real 1789763421] req@ffffa06c07e53800 x1876700024747648/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763446 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763421.559536:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e53800 x1876700024747648/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763446 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763421.559549:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763421.559564:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789763421.559569:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e53800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024747648:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789763421.559580:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024747776:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789763421.559623:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789763421.559641:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763421/real 1789763421] req@ffffa06c07e51c00 x1876700024747776/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763446 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763421.559659:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51c00 x1876700024747776/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763446 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763421.559672:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763421.559677:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789763421.559682:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024747776:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789763421.815019:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763422.070944:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789763422.839048:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763423.094985:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763423.863069:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763424.118942:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789763424.887048:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763425.142975:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763425.911025:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763426.166950:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:1.0:1789763426.679050:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763426.679062:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789763426.679069:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763426.679083:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2860 00000100:00080000:1.0:1789763426.679096:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763426.679128:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789763426.679178:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2d180 x1876700024747904/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763426.679219:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763426.679225:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763426.679228:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763426.679235:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2860 00000100:00080000:1.0:1789763426.679257:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763426.679265:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789763426.679279:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763426.679287:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024747904:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789763426.679309:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2c000 x1876700024748032/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763426.679335:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763426.679340:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763426.679344:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763426.679351:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2860 00000100:00080000:1.0:1789763426.679358:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763426.679364:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789763426.679371:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1789763426.679381:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789763426.679390:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2e300 x1876700024748160/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763426.679444:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763426/real 1789763426] req@ffffa06c08e2d180 x1876700024747904/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763451 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763426.679470:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2d180 x1876700024747904/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763451 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763426.679484:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763426.679489:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789763426.679506:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024747904:192.168.203.107@tcp:250:kworker.0 00000100:00100000:2.0:1789763426.679537:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024748032:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789763426.679570:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789763426.679584:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763426/real 1789763426] req@ffffa06c08e2c000 x1876700024748032/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763451 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763426.679600:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2c000 x1876700024748032/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763451 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763426.679611:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763426.679615:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789763426.679619:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024748032:192.168.203.107@tcp:38:kworker.0 00000100:00100000:2.0:1789763426.679626:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024748160:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789763426.679644:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789763426.679655:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763426/real 1789763426] req@ffffa06c08e2e300 x1876700024748160/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763451 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763426.679671:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2e300 x1876700024748160/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763451 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763426.679682:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763426.679686:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789763426.679693:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024748160:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789763426.935029:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763427.190942:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789763427.959153:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763428.214948:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763428.983014:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763429.238957:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763430.007025:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763430.262956:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789763431.031124:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763431.286955:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789763431.799051:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763431.799089:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763431.799119:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763431.799134:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2865 00000100:00080000:2.0:1789763431.799149:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763431.799159:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789763431.799205:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e53480 x1876700024748288/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763431.799248:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763431.799253:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763431.799255:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763431.799261:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2865 00000100:00080000:2.0:1789763431.799265:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763431.799270:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789763431.799283:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763431.799300:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50a80 x1876700024748416/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763431.799318:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1789763431.799319:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e53480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024748288:192.168.203.107@tcp:250:kworker.0 00000100:00080000:2.0:1789763431.799329:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763431.799335:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763431.799341:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2865 00000100:00080000:2.0:1789763431.799346:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763431.799350:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789763431.799357:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763431.799373:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50700 x1876700024748544/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789763431.799414:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789763431.799469:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763431/real 1789763431] req@ffffa06c07e53480 x1876700024748288/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763456 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763431.799492:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e53480 x1876700024748288/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763456 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763431.799507:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763431.799512:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789763431.799529:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e53480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024748288:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789763431.799560:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024748416:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789763431.799590:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789763431.799606:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763431/real 1789763431] req@ffffa06c07e50a80 x1876700024748416/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763456 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763431.799622:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50a80 x1876700024748416/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763456 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763431.799632:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763431.799636:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789763431.799640:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024748416:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789763431.799647:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024748544:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789763431.799682:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789763431.799696:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763431/real 1789763431] req@ffffa06c07e50700 x1876700024748544/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763456 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763431.799710:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50700 x1876700024748544/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763456 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763431.799720:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763431.799724:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789763431.799728:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024748544:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789763432.055014:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763432.310952:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763433.078959:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763433.334950:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789763434.103040:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763434.358963:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763435.127083:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763435.382947:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763436.150966:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763436.406902:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:1.0:1789763436.919017:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763436.919071:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789763436.919090:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763436.919106:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2870 00000100:00080000:1.0:1789763436.919124:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763436.919136:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789763436.919196:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2d500 x1876700024748672/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763436.919239:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763436.919247:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763436.919251:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763436.919260:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2870 00000100:00080000:1.0:1789763436.919269:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763436.919279:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789763436.919296:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024748672:192.168.203.107@tcp:250:kworker.0 00000008:00100000:1.0:1789763436.919297:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763436.919338:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2e300 x1876700024748800/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763436.919359:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763436.919364:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763436.919367:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763436.919372:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2870 00000100:00080000:1.0:1789763436.919379:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763436.919388:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789763436.919396:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1789763436.919406:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789763436.919419:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2c000 x1876700024748928/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763436.919481:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763436/real 1789763436] req@ffffa06c08e2d500 x1876700024748672/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763461 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763436.919517:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2d500 x1876700024748672/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763461 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763436.919531:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763436.919537:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789763436.919542:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024748672:192.168.203.107@tcp:250:kworker.0 00000100:00100000:2.0:1789763436.919572:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024748800:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789763436.919611:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789763436.919635:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763436/real 1789763436] req@ffffa06c08e2e300 x1876700024748800/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763461 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763436.919661:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2e300 x1876700024748800/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763461 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763436.919692:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763436.919699:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789763436.919706:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024748800:192.168.203.107@tcp:38:kworker.0 00000100:00100000:2.0:1789763436.919718:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024748928:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789763436.919751:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789763436.919781:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763436/real 1789763436] req@ffffa06c08e2c000 x1876700024748928/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763461 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763436.919814:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2c000 x1876700024748928/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763461 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763436.919825:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763436.919830:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789763436.919834:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024748928:192.168.203.107@tcp:38:kworker.0 00010000:00010000:0.0:1789763437.175162:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763437.430963:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763438.199033:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763438.454951:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763439.222987:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763439.478923:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789763440.247052:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763440.502956:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763441.271081:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763441.526952:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789763442.039103:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763442.039115:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763442.039120:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763442.039133:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2875 00000100:00080000:2.0:1789763442.039147:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763442.039156:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789763442.039205:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e52d80 x1876700024749056/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763442.039246:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763442.039251:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763442.039254:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763442.039259:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2875 00000100:00080000:2.0:1789763442.039263:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763442.039269:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789763442.039281:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763442.039297:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e53100 x1876700024749184/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763442.039318:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e52d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024749056:192.168.203.107@tcp:250:kworker.0 00000100:00080000:2.0:1789763442.039350:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763442.039356:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763442.039359:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763442.039367:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2875 00000100:00080000:2.0:1789763442.039373:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763442.039378:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789763442.039387:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763442.039403:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e52a00 x1876700024749312/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789763442.039446:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789763442.039495:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763442/real 1789763442] req@ffffa06c07e52d80 x1876700024749056/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763467 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763442.039516:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e52d80 x1876700024749056/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763467 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763442.039527:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763442.039532:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789763442.039536:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e52d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024749056:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789763442.039564:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e53100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024749184:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789763442.039585:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789763442.039597:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763442/real 1789763442] req@ffffa06c07e53100 x1876700024749184/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763467 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763442.039610:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e53100 x1876700024749184/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763467 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763442.039618:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763442.039634:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789763442.039638:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e53100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024749184:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789763442.039644:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e52a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024749312:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789763442.039679:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789763442.039690:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763442/real 1789763442] req@ffffa06c07e52a00 x1876700024749312/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763467 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763442.039702:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e52a00 x1876700024749312/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763467 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763442.039710:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763442.039713:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789763442.039715:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e52a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024749312:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789763442.294980:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763442.550945:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763443.319060:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763443.574952:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789763444.343192:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763444.598938:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763445.367073:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763445.623017:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763446.391069:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763446.646947:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789763447.158975:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763447.158987:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763447.158992:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763447.159004:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2880 00000100:00080000:2.0:1789763447.159017:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763447.159026:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789763447.159073:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e52a00 x1876700024749440/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763447.159110:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763447.159115:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763447.159118:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763447.159124:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2880 00000100:00080000:2.0:1789763447.159128:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763447.159133:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789763447.159164:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763447.159187:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e52a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024749440:192.168.203.107@tcp:250:kworker.0 00000100:00100000:2.0:1789763447.159188:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e53100 x1876700024749568/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763447.159220:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763447.159237:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763447.159240:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763447.159245:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2880 00000100:00080000:2.0:1789763447.159253:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763447.159259:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789763447.159266:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763447.159281:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e52d80 x1876700024749696/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789763447.159304:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789763447.159359:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763447/real 1789763447] req@ffffa06c07e52a00 x1876700024749440/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763472 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763447.159382:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e52a00 x1876700024749440/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763472 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763447.159397:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763447.159402:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789763447.159407:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e52a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024749440:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789763447.159438:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e53100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024749568:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789763447.159473:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789763447.159498:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763447/real 1789763447] req@ffffa06c07e53100 x1876700024749568/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763472 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763447.159556:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e53100 x1876700024749568/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763472 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763447.159573:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763447.159578:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789763447.159582:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e53100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024749568:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789763447.159591:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e52d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024749696:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789763447.159624:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789763447.159639:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763447/real 1789763447] req@ffffa06c07e52d80 x1876700024749696/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763472 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763447.159671:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e52d80 x1876700024749696/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763472 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763447.159681:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763447.159686:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789763447.159689:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e52d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024749696:192.168.203.107@tcp:38:kworker.0 00010000:00010000:1.0:1789763447.415072:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763447.670945:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763448.439041:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763448.694987:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763449.463028:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763449.718970:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789763450.487055:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763450.743012:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763451.511069:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763451.766996:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789763452.278927:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763452.278937:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763452.278941:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763452.278950:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2886 00000100:00080000:2.0:1789763452.278961:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763452.278968:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789763452.279005:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50700 x1876700024749824/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763452.279032:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763452.279036:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763452.279053:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763452.279057:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2886 00000100:00080000:2.0:1789763452.279062:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763452.279066:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789763452.279076:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763452.279089:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50a80 x1876700024749952/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763452.279102:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1789763452.279103:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024749824:192.168.203.107@tcp:250:kworker.0 00000100:00080000:2.0:1789763452.279109:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763452.279114:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763452.279119:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2886 00000100:00080000:2.0:1789763452.279122:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763452.279125:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789763452.279129:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763452.279140:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e53480 x1876700024750080/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789763452.279176:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789763452.279215:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763452/real 1789763452] req@ffffa06c07e50700 x1876700024749824/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763477 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763452.279231:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50700 x1876700024749824/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763477 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763452.279250:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763452.279254:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789763452.279258:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024749824:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789763452.279280:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024749952:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789763452.279301:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789763452.279312:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763452/real 1789763452] req@ffffa06c07e50a80 x1876700024749952/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763477 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763452.279322:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50a80 x1876700024749952/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763477 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763452.279330:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763452.279333:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789763452.279336:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024749952:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789763452.279340:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e53480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024750080:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789763452.279353:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789763452.279361:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763452/real 1789763452] req@ffffa06c07e53480 x1876700024750080/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763477 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763452.279370:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e53480 x1876700024750080/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763477 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763452.279376:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763452.279379:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789763452.279381:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e53480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024750080:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789763452.535031:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763452.791024:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789763453.559076:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763453.814996:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763454.583061:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763454.838929:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763455.607117:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763455.863043:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789763456.631110:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763456.887071:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:1.0:1789763457.399043:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763457.399057:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789763457.399062:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763457.399075:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2891 00000100:00080000:1.0:1789763457.399090:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763457.399100:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789763457.399203:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2d180 x1876700024750208/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763457.399245:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763457.399250:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763457.399252:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763457.399257:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2891 00000100:00080000:1.0:1789763457.399262:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763457.399267:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789763457.399299:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763457.399318:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2c700 x1876700024750336/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763457.399338:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024750208:192.168.203.107@tcp:250:kworker.0 00000100:00080000:1.0:1789763457.399347:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763457.399375:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763457.399380:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763457.399392:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2891 00000100:00080000:1.0:1789763457.399403:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763457.399412:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000800:00000100:2.0:1789763457.399433:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000008:00100000:1.0:1789763457.399452:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763457.399498:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763457/real 1789763457] req@ffffa06c08e2d180 x1876700024750208/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763482 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763457.399500:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2ed80 x1876700024750464/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763457.399573:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2d180 x1876700024750208/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763482 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763457.399616:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763457.399623:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789763457.399629:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024750208:192.168.203.107@tcp:250:kworker.0 00000100:00100000:2.0:1789763457.399698:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024750336:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789763457.399737:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789763457.399756:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763457/real 1789763457] req@ffffa06c08e2c700 x1876700024750336/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763482 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763457.399772:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2c700 x1876700024750336/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763482 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763457.399783:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763457.399788:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789763457.399792:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024750336:192.168.203.107@tcp:38:kworker.0 00000100:00100000:2.0:1789763457.399800:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024750464:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789763457.399821:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789763457.399832:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763457/real 1789763457] req@ffffa06c08e2ed80 x1876700024750464/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763482 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763457.399911:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2ed80 x1876700024750464/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763482 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763457.399928:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763457.399935:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789763457.399943:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024750464:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789763457.655080:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763457.910993:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763458.679063:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763458.934991:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789763459.703126:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763459.958976:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763460.727038:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763460.982952:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763461.751034:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763462.006971:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:1.0:1789763462.518979:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763462.518998:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789763462.519005:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763462.519021:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2896 00000100:00080000:1.0:1789763462.519038:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763462.519047:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789763462.519119:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2ca80 x1876700024750592/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763462.519155:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763462.519160:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763462.519162:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763462.519168:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2896 00000100:00080000:1.0:1789763462.519172:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763462.519176:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789763462.519189:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763462.519208:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2ed80 x1876700024750720/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763462.519225:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763462.519230:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763462.519233:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763462.519238:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2896 00000100:00100000:2.0:1789763462.519247:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024750592:192.168.203.107@tcp:250:kworker.0 00000100:00080000:1.0:1789763462.519266:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763462.519279:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789763462.519290:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763462.519320:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2c700 x1876700024750848/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789763462.519340:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789763462.519420:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763462/real 1789763462] req@ffffa06c08e2ca80 x1876700024750592/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763487 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763462.519460:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2ca80 x1876700024750592/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763487 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763462.519479:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763462.519486:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789763462.519491:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024750592:192.168.203.107@tcp:250:kworker.0 00000100:00100000:2.0:1789763462.519502:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024750720:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789763462.519549:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789763462.519572:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763462/real 1789763462] req@ffffa06c08e2ed80 x1876700024750720/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763487 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763462.519594:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2ed80 x1876700024750720/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763487 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763462.519606:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763462.519614:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789763462.519621:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024750720:192.168.203.107@tcp:38:kworker.0 00000100:00100000:2.0:1789763462.519658:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024750848:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789763462.519688:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789763462.519700:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763462/real 1789763462] req@ffffa06c08e2c700 x1876700024750848/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763487 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763462.519733:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2c700 x1876700024750848/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763487 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763462.519745:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763462.519750:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789763462.519755:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024750848:192.168.203.107@tcp:38:kworker.0 00010000:00010000:1.0:1789763462.775107:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763463.030977:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763463.799025:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763464.054985:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789763464.823065:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763465.079000:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789763465.847009:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763466.102933:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763466.871055:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763467.126983:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:1.0:1789763467.638957:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763467.638970:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789763467.638976:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763467.638989:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2901 00000100:00080000:1.0:1789763467.639003:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763467.639013:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789763467.639063:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2d180 x1876700024750976/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763467.639100:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763467.639105:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763467.639108:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763467.639113:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2901 00000100:00080000:1.0:1789763467.639118:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763467.639122:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789763467.639133:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763467.639150:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2c000 x1876700024751104/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763467.639182:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024750976:192.168.203.107@tcp:250:kworker.0 00000100:00080000:1.0:1789763467.639211:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763467.639216:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763467.639235:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763467.639240:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2901 00000100:00080000:1.0:1789763467.639243:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763467.639247:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789763467.639251:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763467.639262:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2e300 x1876700024751232/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789763467.639281:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789763467.639370:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763467/real 1789763467] req@ffffa06c08e2d180 x1876700024750976/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763492 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763467.639395:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2d180 x1876700024750976/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763492 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763467.639409:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763467.639414:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789763467.639435:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024750976:192.168.203.107@tcp:250:kworker.0 00000100:00100000:2.0:1789763467.639456:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024751104:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789763467.639474:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789763467.639501:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763467/real 1789763467] req@ffffa06c08e2c000 x1876700024751104/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763492 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763467.639517:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2c000 x1876700024751104/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763492 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763467.639548:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763467.639552:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789763467.639557:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024751104:192.168.203.107@tcp:38:kworker.0 00000100:00100000:2.0:1789763467.639565:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024751232:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789763467.639591:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789763467.639604:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763467/real 1789763467] req@ffffa06c08e2e300 x1876700024751232/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763492 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763467.639618:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2e300 x1876700024751232/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763492 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763467.639627:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763467.639630:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789763467.639634:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024751232:192.168.203.107@tcp:38:kworker.0 00010000:00010000:2.0:1789763467.894931:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763468.150908:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763468.919002:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763469.174947:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789763469.943029:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763470.198977:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763470.967132:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763471.222924:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763471.991001:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763472.246942:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:1.0:1789763472.758964:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763472.758978:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789763472.758983:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763472.758996:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2906 00000100:00080000:1.0:1789763472.759010:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763472.759019:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789763472.759069:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2d500 x1876700024751360/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763472.759106:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763472.759111:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763472.759113:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763472.759118:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2906 00000100:00080000:1.0:1789763472.759122:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763472.759126:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789763472.759158:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763472.759184:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2e300 x1876700024751488/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763472.759194:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024751360:192.168.203.107@tcp:250:kworker.0 00000100:00080000:1.0:1789763472.759224:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763472.759229:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763472.759232:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763472.759238:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2906 00000100:00080000:1.0:1789763472.759244:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763472.759249:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789763472.759257:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763472.759272:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2c000 x1876700024751616/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789763472.759294:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789763472.759365:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763472/real 1789763472] req@ffffa06c08e2d500 x1876700024751360/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763497 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763472.759405:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2d500 x1876700024751360/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763497 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763472.759426:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763472.759434:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789763472.759457:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024751360:192.168.203.107@tcp:250:kworker.0 00000100:00100000:2.0:1789763472.759495:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024751488:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789763472.759530:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789763472.759547:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763472/real 1789763472] req@ffffa06c08e2e300 x1876700024751488/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763497 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763472.759563:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2e300 x1876700024751488/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763497 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763472.759574:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763472.759578:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789763472.759582:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024751488:192.168.203.107@tcp:38:kworker.0 00000100:00100000:2.0:1789763472.759590:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024751616:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789763472.759610:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789763472.759621:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763472/real 1789763472] req@ffffa06c08e2c000 x1876700024751616/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763497 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763472.759634:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2c000 x1876700024751616/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763497 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763472.759644:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763472.759647:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789763472.759651:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024751616:192.168.203.107@tcp:38:kworker.0 00010000:00010000:0.0:1789763473.015028:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763473.270932:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763474.038978:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763474.294990:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789763475.063072:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763475.318976:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789763476.087108:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763476.342948:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763477.110985:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763477.366945:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789763477.879022:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763477.879035:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763477.879060:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763477.879073:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2911 00000100:00080000:2.0:1789763477.879087:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763477.879096:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789763477.879142:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51c00 x1876700024751744/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763477.879190:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763477.879195:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763477.879198:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763477.879203:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2911 00000100:00080000:2.0:1789763477.879208:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763477.879219:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789763477.879231:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763477.879248:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e53800 x1876700024751872/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763477.879266:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1789763477.879271:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024751744:192.168.203.107@tcp:250:kworker.0 00000100:00080000:2.0:1789763477.879273:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763477.879280:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763477.879292:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2911 00000100:00080000:2.0:1789763477.879296:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763477.879301:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789763477.879307:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763477.879322:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e52680 x1876700024752000/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789763477.879367:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789763477.879432:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763477/real 1789763477] req@ffffa06c07e51c00 x1876700024751744/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763502 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763477.879456:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51c00 x1876700024751744/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763502 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763477.879470:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763477.879475:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789763477.879480:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024751744:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789763477.879519:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e53800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024751872:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789763477.879564:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789763477.879580:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763477/real 1789763477] req@ffffa06c07e53800 x1876700024751872/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763502 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763477.879595:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e53800 x1876700024751872/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763502 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763477.879606:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763477.879609:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789763477.879613:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e53800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024751872:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789763477.879621:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e52680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024752000:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789763477.879640:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789763477.879651:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763477/real 1789763477] req@ffffa06c07e52680 x1876700024752000/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763502 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763477.879665:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e52680 x1876700024752000/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763502 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763477.879685:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763477.879688:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789763477.879691:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e52680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024752000:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789763478.134960:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763478.390914:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789763479.159092:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763479.414955:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763480.183038:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763480.438999:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763481.207006:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763481.463000:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789763482.231075:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763482.486982:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:1.0:1789763482.999003:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763482.999020:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789763482.999040:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763482.999065:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2916 00000100:00080000:1.0:1789763482.999082:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763482.999122:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789763482.999194:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2d180 x1876700024752128/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763482.999234:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763482.999244:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763482.999247:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763482.999253:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2916 00000100:00080000:1.0:1789763482.999260:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763482.999266:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789763482.999279:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763482.999300:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2c700 x1876700024752256/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763482.999326:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024752128:192.168.203.107@tcp:250:kworker.0 00000100:00080000:1.0:1789763482.999329:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763482.999342:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763482.999349:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763482.999356:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2916 00000100:00080000:1.0:1789763482.999360:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763482.999364:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789763482.999370:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763482.999394:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2ed80 x1876700024752384/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763482.999511:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024752256:192.168.203.107@tcp:38:kworker.0 00000100:00100000:2.0:1789763482.999553:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024752384:192.168.203.107@tcp:38:kworker.0 00010000:00010000:0.0:1789763483.255059:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763483.510948:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763484.278976:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763484.535030:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789763485.303185:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763485.558979:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000400:00000100:3.0:1789763486.071174:0:2330:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.107:988 00000400:00000100:3.0:1789763486.071247:0:2330:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.107@tcp at host 192.168.203.107 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1789763486.071268:0:2330:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.107@tcp 00000800:00000100:3.0:1789763486.071321:0:2330:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.107@tcp 00000800:00000100:3.0:1789763486.071337:0:2330:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.107@tcp 00000100:00100000:2.0:1789763486.071386:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763482/real 1789763486] req@ffffa06c08e2c700 x1876700024752256/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763507 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763486.071411:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2c700 x1876700024752256/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763507 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763486.071420:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763486.071424:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789763486.071427:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024752256:192.168.203.107@tcp:38:kworker.0 00000100:00100000:2.0:1789763486.071433:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763482/real 1789763486] req@ffffa06c08e2ed80 x1876700024752384/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763507 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763486.071445:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2ed80 x1876700024752384/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763507 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763486.071454:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763486.071458:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789763486.071462:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024752384:192.168.203.107@tcp:38:kworker.0 00000100:00100000:2.0:1789763486.071487:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763482/real 1789763486] req@ffffa06c08e2d180 x1876700024752128/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763507 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763486.071502:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2d180 x1876700024752128/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763507 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763486.071512:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763486.071516:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789763486.071519:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024752128:192.168.203.107@tcp:250:kworker.0 00010000:00010000:0.0:1789763486.327112:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763486.582980:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763487.351029:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763487.606992:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789763488.119058:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763488.119073:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763488.119080:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763488.119112:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2921 00000100:00080000:2.0:1789763488.119129:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763488.119141:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789763488.119194:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50380 x1876700024752512/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763488.119240:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763488.119245:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763488.119248:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763488.119253:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2921 00000100:00080000:2.0:1789763488.119258:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763488.119263:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789763488.119274:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763488.119289:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e53b80 x1876700024752640/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763488.119326:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763488.119334:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:1.0:1789763488.119336:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024752512:192.168.203.107@tcp:250:kworker.0 00000100:00080000:2.0:1789763488.119338:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763488.119352:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2921 00000100:00080000:2.0:1789763488.119359:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763488.119364:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789763488.119371:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763488.119390:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51f80 x1876700024752768/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789763488.119447:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789763488.119559:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763488/real 1789763488] req@ffffa06c07e50380 x1876700024752512/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763513 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763488.119593:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50380 x1876700024752512/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763513 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763488.119613:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763488.119620:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789763488.119626:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024752512:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789763488.119693:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e53b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024752640:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789763488.119739:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789763488.119765:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763488/real 1789763488] req@ffffa06c07e53b80 x1876700024752640/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763513 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763488.119789:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e53b80 x1876700024752640/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763513 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763488.119815:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763488.119822:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789763488.119829:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e53b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024752640:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789763488.119841:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024752768:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789763488.119940:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789763488.119964:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763488/real 1789763488] req@ffffa06c07e51f80 x1876700024752768/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763513 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763488.119987:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51f80 x1876700024752768/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763513 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763488.120026:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763488.120037:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789763488.120046:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024752768:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789763488.375040:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763488.631019:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789763489.399073:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763489.654949:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763490.423066:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763490.678965:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763491.447072:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763491.702993:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789763492.471050:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763492.726961:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789763493.238972:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763493.238984:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763493.238989:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763493.239000:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2927 00000100:00080000:2.0:1789763493.239024:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763493.239033:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789763493.239079:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50e00 x1876700024752896/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763493.239114:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763493.239120:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763493.239124:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763493.239130:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2927 00000100:00080000:2.0:1789763493.239137:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763493.239145:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789763493.239174:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024752896:192.168.203.107@tcp:250:kworker.0 00000008:00100000:2.0:1789763493.239185:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763493.239215:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51500 x1876700024753024/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763493.239232:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763493.239236:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763493.239239:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763493.239243:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2927 00000100:00080000:2.0:1789763493.239249:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763493.239253:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789763493.239260:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1789763493.239266:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789763493.239280:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50000 x1876700024753152/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763493.239319:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763493/real 1789763493] req@ffffa06c07e50e00 x1876700024752896/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763518 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763493.239342:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50e00 x1876700024752896/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763518 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763493.239355:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763493.239359:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789763493.239364:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024752896:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789763493.239394:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024753024:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789763493.239432:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789763493.239455:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763493/real 1789763493] req@ffffa06c07e51500 x1876700024753024/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763518 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763493.239477:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51500 x1876700024753024/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763518 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763493.239487:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763493.239491:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789763493.239505:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024753024:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789763493.239513:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024753152:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789763493.239540:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789763493.239552:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763493/real 1789763493] req@ffffa06c07e50000 x1876700024753152/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763518 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763493.239565:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50000 x1876700024753152/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763518 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763493.239574:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763493.239577:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789763493.239581:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024753152:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789763493.494998:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763493.750962:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763494.519068:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763494.774957:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789763495.543038:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763495.798967:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763496.567008:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763496.822949:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763497.590975:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763497.846964:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789763498.359036:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763498.359047:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763498.359053:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763498.359067:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2932 00000100:00080000:0.0:1789763498.359082:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763498.359093:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789763498.359155:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d3b80 x1876700024753280/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763498.359192:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763498.359198:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763498.359201:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763498.359207:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2932 00000100:00080000:0.0:1789763498.359213:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763498.359219:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789763498.359233:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763498.359255:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024753280:192.168.203.107@tcp:250:kworker.0 00000100:00100000:0.0:1789763498.359261:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0000 x1876700024753408/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763498.359286:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763498.359292:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763498.359296:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763498.359315:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2932 00000100:00080000:0.0:1789763498.359325:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763498.359333:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789763498.359342:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763498.359364:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d2300 x1876700024753536/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789763498.359370:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789763498.359431:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763498/real 1789763498] req@ffffa06c091d3b80 x1876700024753280/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763523 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763498.359458:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d3b80 x1876700024753280/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763523 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763498.359472:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763498.359477:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789763498.359482:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024753280:192.168.203.107@tcp:250:kworker.0 00000100:00100000:2.0:1789763498.359511:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024753408:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789763498.359559:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789763498.359605:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763498/real 1789763498] req@ffffa06c091d0000 x1876700024753408/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763523 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763498.359623:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0000 x1876700024753408/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763523 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763498.359634:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763498.359638:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789763498.359643:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024753408:192.168.203.107@tcp:38:kworker.0 00000100:00100000:2.0:1789763498.359651:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024753536:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789763498.359674:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789763498.359687:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763498/real 1789763498] req@ffffa06c091d2300 x1876700024753536/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763523 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763498.359701:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d2300 x1876700024753536/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763523 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763498.359710:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763498.359714:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789763498.359717:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024753536:192.168.203.107@tcp:38:kworker.0 00010000:00010000:0.0:1789763498.615125:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763498.870998:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789763499.638985:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763499.894986:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763500.663017:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763500.919023:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789763501.687015:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763501.942987:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789763502.711053:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763502.967010:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789763503.479068:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763503.479082:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763503.479087:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763503.479099:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2937 00000100:00080000:2.0:1789763503.479113:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763503.479143:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789763503.479188:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51180 x1876700024753664/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763503.479229:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763503.479234:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763503.479236:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763503.479242:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2937 00000100:00080000:2.0:1789763503.479247:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763503.479252:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789763503.479272:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763503.479287:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e52300 x1876700024753792/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763503.479302:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763503.479307:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763503.479310:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763503.479317:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2937 00000100:00080000:2.0:1789763503.479322:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763503.479327:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789763503.479333:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763503.479345:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51880 x1876700024753920/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763503.479354:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024753664:192.168.203.107@tcp:250:kworker.0 00000800:00000100:1.0:1789763503.479452:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789763503.479501:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763503/real 1789763503] req@ffffa06c07e51180 x1876700024753664/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763528 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763503.479524:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51180 x1876700024753664/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763528 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763503.479565:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763503.479573:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789763503.479578:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024753664:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789763503.479604:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e52300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024753792:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789763503.479636:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789763503.479653:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763503/real 1789763503] req@ffffa06c07e52300 x1876700024753792/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763528 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763503.479669:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e52300 x1876700024753792/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763528 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763503.479679:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763503.479683:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789763503.479688:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e52300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024753792:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789763503.479716:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024753920:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789763503.479735:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789763503.479747:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763503/real 1789763503] req@ffffa06c07e51880 x1876700024753920/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763528 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763503.479761:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51880 x1876700024753920/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763528 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763503.479770:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763503.479774:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789763503.479777:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024753920:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789763503.735053:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763503.990991:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763504.759032:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763505.014985:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789763505.783110:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763506.038969:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763506.807026:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763507.062938:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763507.831036:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763508.087004:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789763508.599041:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763508.599054:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763508.599060:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763508.599074:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2942 00000100:00080000:0.0:1789763508.599089:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763508.599099:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789763508.599147:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d1500 x1876700024754048/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763508.599197:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763508.599202:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763508.599204:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763508.599209:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2942 00000100:00080000:0.0:1789763508.599214:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763508.599219:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789763508.599231:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763508.599248:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0e00 x1876700024754176/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763508.599256:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024754048:192.168.203.107@tcp:250:kworker.0 00000100:00080000:0.0:1789763508.599288:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763508.599293:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763508.599296:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763508.599303:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2942 00000100:00080000:0.0:1789763508.599333:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763508.599339:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000800:00000100:1.0:1789763508.599348:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000008:00100000:0.0:1789763508.599353:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763508.599376:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d2a00 x1876700024754304/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763508.599435:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763508/real 1789763508] req@ffffa06c091d1500 x1876700024754048/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763533 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763508.599462:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d1500 x1876700024754048/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763533 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763508.599477:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763508.599482:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789763508.599488:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024754048:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789763508.599545:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024754176:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789763508.599583:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789763508.599599:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763508/real 1789763508] req@ffffa06c091d0e00 x1876700024754176/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763533 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763508.599615:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0e00 x1876700024754176/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763533 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763508.599626:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763508.599630:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789763508.599634:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024754176:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789763508.599641:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024754304:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789763508.599660:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789763508.599672:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763508/real 1789763508] req@ffffa06c091d2a00 x1876700024754304/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763533 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763508.599685:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d2a00 x1876700024754304/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763533 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763508.599713:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763508.599717:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789763508.599722:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024754304:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789763508.855080:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789763509.111018:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763509.879029:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789763510.134975:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763510.903047:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789763511.158993:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763511.927051:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789763512.182938:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763512.951099:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789763513.207014:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789763513.718982:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763513.718999:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763513.719006:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763513.719023:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2947 00000100:00080000:0.0:1789763513.719041:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763513.719055:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789763513.719113:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d2680 x1876700024754432/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763513.719160:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763513.719166:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763513.719189:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1789763513.719200:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024754432:192.168.203.107@tcp:250:kworker.0 00000100:00080000:0.0:1789763513.719204:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2947 00000100:00080000:0.0:1789763513.719220:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763513.719227:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789763513.719241:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763513.719265:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0380 x1876700024754560/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763513.719284:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763513.719289:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763513.719291:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763513.719296:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2947 00000100:00080000:0.0:1789763513.719302:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1789763513.719311:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00080000:0.0:1789763513.719313:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789763513.719325:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763513.719340:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d3100 x1876700024754688/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763513.719394:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763513/real 1789763513] req@ffffa06c091d2680 x1876700024754432/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763538 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763513.719432:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d2680 x1876700024754432/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763538 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763513.719452:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763513.719487:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789763513.719496:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024754432:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789763513.719539:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024754560:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789763513.719576:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789763513.719594:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763513/real 1789763513] req@ffffa06c091d0380 x1876700024754560/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763538 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763513.719632:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0380 x1876700024754560/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763538 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763513.719650:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763513.719655:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789763513.719660:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024754560:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789763513.719685:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024754688:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789763513.719718:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789763513.719733:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763513/real 1789763513] req@ffffa06c091d3100 x1876700024754688/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763538 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763513.719747:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d3100 x1876700024754688/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763538 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763513.719757:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763513.719760:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789763513.719764:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024754688:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789763513.975064:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789763514.230963:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763514.999036:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763515.255060:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763516.023036:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763516.279009:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763517.047076:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763517.302975:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789763518.071095:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763518.326964:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789763518.838979:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763518.838996:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763518.839003:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763518.839019:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2952 00000100:00080000:0.0:1789763518.839038:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763518.839050:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789763518.839108:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d1f80 x1876700024754816/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763518.839167:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763518.839173:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763518.839175:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763518.839181:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2952 00000100:00080000:0.0:1789763518.839187:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763518.839192:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789763518.839206:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763518.839225:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0a80 x1876700024754944/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763518.839242:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1789763518.839245:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024754816:192.168.203.107@tcp:250:kworker.0 00000100:00080000:0.0:1789763518.839249:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763518.839256:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763518.839264:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2952 00000100:00080000:0.0:1789763518.839268:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763518.839272:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789763518.839280:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763518.839296:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0700 x1876700024755072/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789763518.839339:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789763518.839397:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763518/real 1789763518] req@ffffa06c091d1f80 x1876700024754816/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763543 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763518.839461:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d1f80 x1876700024754816/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763543 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763518.839476:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763518.839482:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789763518.839487:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024754816:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789763518.839516:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024754944:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789763518.839548:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789763518.839579:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763518/real 1789763518] req@ffffa06c091d0a80 x1876700024754944/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763543 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763518.839595:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0a80 x1876700024754944/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763543 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763518.839606:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763518.839611:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789763518.839615:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024754944:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789763518.839622:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024755072:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789763518.839645:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789763518.839657:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763518/real 1789763518] req@ffffa06c091d0700 x1876700024755072/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763543 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763518.839670:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0700 x1876700024755072/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763543 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763518.839680:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763518.839684:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789763518.839687:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024755072:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789763519.095102:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763519.350965:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789763520.118984:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763520.374928:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789763521.143059:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763521.398950:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763522.166972:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763522.422950:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789763523.191072:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763523.446961:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:3.0:1789763523.959075:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763523.959088:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763523.959094:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763523.959107:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2957 00000100:00080000:3.0:1789763523.959124:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763523.959136:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:3.0:1789763523.959196:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872d180 x1876700024755200/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763523.959250:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763523.959255:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763523.959258:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763523.959264:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2957 00000100:00080000:3.0:1789763523.959270:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763523.959297:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789763523.959309:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763523.959329:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872ca80 x1876700024755328/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763523.959347:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763523.959352:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763523.959355:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763523.959360:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2957 00000100:00080000:3.0:1789763523.959366:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763523.959371:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789763523.959381:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024755200:192.168.203.107@tcp:250:kworker.0 00000008:00100000:3.0:1789763523.959381:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763523.959448:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872f100 x1876700024755456/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789763523.959542:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789763523.959594:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763523/real 1789763523] req@ffffa06c1872d180 x1876700024755200/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763548 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763523.959641:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872d180 x1876700024755200/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763548 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763523.959655:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763523.959660:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789763523.959665:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024755200:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789763523.959675:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024755328:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789763523.959722:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789763523.959737:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763523/real 1789763523] req@ffffa06c1872ca80 x1876700024755328/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763548 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763523.959751:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872ca80 x1876700024755328/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763548 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763523.959761:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763523.959766:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789763523.959772:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024755328:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789763523.959811:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024755456:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789763523.959849:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789763523.959871:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763523/real 1789763523] req@ffffa06c1872f100 x1876700024755456/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763548 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763523.959934:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872f100 x1876700024755456/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763548 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763523.959962:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763523.959967:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789763523.959971:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024755456:192.168.203.107@tcp:38:kworker.0 00010000:00010000:0.0:1789763524.215086:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763524.470943:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763525.239079:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763525.495082:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789763526.263072:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763526.518955:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789763527.287050:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789763527.543032:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763528.310943:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789763528.566938:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789763529.078969:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763529.078994:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763529.078998:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763529.079006:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2962 00000100:00080000:0.0:1789763529.079016:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763529.079034:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789763529.079076:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0700 x1876700024755584/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763529.079101:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763529.079105:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763529.079107:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763529.079110:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2962 00000100:00080000:0.0:1789763529.079113:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763529.079116:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789763529.079125:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763529.079136:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0a80 x1876700024755712/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763529.079147:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763529.079151:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763529.079154:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1789763529.079156:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024755584:192.168.203.107@tcp:250:kworker.0 00000100:00080000:0.0:1789763529.079160:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2962 00000100:00080000:0.0:1789763529.079166:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763529.079171:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789763529.079175:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763529.079184:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d1f80 x1876700024755840/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789763529.079227:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789763529.079268:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763529/real 1789763529] req@ffffa06c091d0700 x1876700024755584/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763554 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763529.079291:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0700 x1876700024755584/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763554 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763529.079305:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763529.079311:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789763529.079316:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024755584:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789763529.079346:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024755712:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789763529.079393:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789763529.079409:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763529/real 1789763529] req@ffffa06c091d0a80 x1876700024755712/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763554 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763529.079424:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0a80 x1876700024755712/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763554 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763529.079434:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763529.079449:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789763529.079453:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024755712:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789763529.079461:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024755840:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789763529.079487:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789763529.079501:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763529/real 1789763529] req@ffffa06c091d1f80 x1876700024755840/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763554 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763529.079517:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d1f80 x1876700024755840/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763554 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763529.079527:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763529.079531:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789763529.079536:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024755840:192.168.203.107@tcp:38:kworker.0 00010000:00010000:1.0:1789763529.335068:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789763529.590988:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789763530.359043:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789763530.614998:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763531.383029:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789763531.638989:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789763532.407001:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789763532.662944:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789763533.431077:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789763533.686948:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789763534.199024:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763534.199040:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763534.199047:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763534.199085:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2968 00000100:00080000:0.0:1789763534.199103:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763534.199116:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789763534.199195:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d3100 x1876700024755968/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763534.199255:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763534.199265:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763534.199271:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763534.199282:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2968 00000100:00080000:0.0:1789763534.199296:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763534.199335:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789763534.199378:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763534.199411:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024755968:192.168.203.107@tcp:250:kworker.0 00000100:00100000:0.0:1789763534.199413:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0380 x1876700024756096/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763534.199445:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763534.199474:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763534.199478:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763534.199490:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2968 00000800:00000100:2.0:1789763534.199535:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00080000:0.0:1789763534.199617:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763534.199632:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789763534.199642:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763534/real 1789763534] req@ffffa06c091d3100 x1876700024755968/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763559 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1789763534.199646:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763534.199680:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d3100 x1876700024755968/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763559 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763534.199691:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d2680 x1876700024756224/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763534.199699:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763534.199715:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789763534.199726:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024755968:192.168.203.107@tcp:250:kworker.0 00000100:00100000:2.0:1789763534.199773:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024756096:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789763534.199808:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789763534.199830:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763534/real 1789763534] req@ffffa06c091d0380 x1876700024756096/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763559 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763534.199930:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0380 x1876700024756096/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763559 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763534.199964:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763534.199975:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789763534.199991:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024756096:192.168.203.107@tcp:38:kworker.0 00000100:00100000:2.0:1789763534.200020:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024756224:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789763534.200089:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789763534.200110:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763534/real 1789763534] req@ffffa06c091d2680 x1876700024756224/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763559 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763534.200128:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d2680 x1876700024756224/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763559 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763534.200159:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763534.200164:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789763534.200169:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024756224:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789763534.455079:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789763534.710947:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789763535.479043:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763535.735037:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789763536.503274:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763536.758973:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763537.527070:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763537.783019:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789763538.551021:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763538.807013:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789763539.319014:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763539.319041:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763539.319070:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763539.319087:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2973 00000100:00080000:0.0:1789763539.319103:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763539.319117:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789763539.319185:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d2a00 x1876700024756352/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763539.319251:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763539.319259:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763539.319264:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763539.319296:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2973 00000100:00080000:0.0:1789763539.319306:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763539.319315:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789763539.319365:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763539.319376:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024756352:192.168.203.107@tcp:250:kworker.0 00000100:00100000:0.0:1789763539.319402:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0e00 x1876700024756480/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763539.319432:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763539.319439:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763539.319444:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763539.319453:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2973 00000100:00080000:0.0:1789763539.319464:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1789763539.319469:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00080000:0.0:1789763539.319481:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789763539.319494:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763539.319524:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d1500 x1876700024756608/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763539.319532:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763539/real 1789763539] req@ffffa06c091d2a00 x1876700024756352/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763564 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763539.319587:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d2a00 x1876700024756352/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763564 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763539.319602:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763539.319608:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789763539.319615:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024756352:192.168.203.107@tcp:250:kworker.0 00000100:00100000:2.0:1789763539.319646:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024756480:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789763539.319672:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789763539.319686:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763539/real 1789763539] req@ffffa06c091d0e00 x1876700024756480/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763564 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763539.319701:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0e00 x1876700024756480/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763564 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763539.319712:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763539.319716:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789763539.319733:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024756480:192.168.203.107@tcp:38:kworker.0 00000100:00100000:2.0:1789763539.319758:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024756608:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789763539.319782:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789763539.319793:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763539/real 1789763539] req@ffffa06c091d1500 x1876700024756608/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763564 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763539.319806:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d1500 x1876700024756608/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763564 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763539.319832:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763539.319836:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789763539.319840:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024756608:192.168.203.107@tcp:38:kworker.0 00010000:00010000:2.0:1789763539.575059:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763539.830945:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763540.599030:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763540.854994:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789763541.623097:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763541.878952:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789763542.646988:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763542.902962:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789763543.671123:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763543.926984:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789763544.439050:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763544.439063:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763544.439068:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763544.439081:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2978 00000100:00080000:2.0:1789763544.439095:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763544.439104:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789763544.439155:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51880 x1876700024756736/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763544.439199:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763544.439204:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763544.439226:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763544.439233:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2978 00000100:00080000:2.0:1789763544.439260:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763544.439267:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789763544.439281:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763544.439303:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e52300 x1876700024756864/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763544.439322:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763544.439326:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763544.439328:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763544.439333:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2978 00000100:00080000:2.0:1789763544.439336:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763544.439342:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789763544.439349:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763544.439366:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024756736:192.168.203.107@tcp:250:kworker.0 00000100:00100000:2.0:1789763544.439369:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51180 x1876700024756992/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789763544.439516:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789763544.439585:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763544/real 1789763544] req@ffffa06c07e51880 x1876700024756736/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763569 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763544.439613:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51880 x1876700024756736/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763569 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763544.439630:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763544.439638:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789763544.439669:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024756736:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789763544.439719:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e52300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024756864:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789763544.439763:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789763544.439783:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763544/real 1789763544] req@ffffa06c07e52300 x1876700024756864/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763569 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763544.439800:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e52300 x1876700024756864/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763569 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763544.439812:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763544.439817:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789763544.439821:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e52300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024756864:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789763544.439829:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024756992:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789763544.439901:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789763544.439917:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763544/real 1789763544] req@ffffa06c07e51180 x1876700024756992/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763569 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763544.439932:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51180 x1876700024756992/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763569 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763544.439942:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763544.439947:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789763544.439951:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024756992:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789763544.695091:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763544.951100:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789763545.719049:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763545.974949:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763546.743037:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763546.998994:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763547.766983:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763548.022915:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763548.791027:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763549.046943:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:1.0:1789763549.558948:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763549.558958:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789763549.558973:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763549.558982:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2983 00000100:00080000:1.0:1789763549.558993:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763549.559002:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789763549.559070:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2ca80 x1876700024757120/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763549.559129:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763549.559132:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763549.559134:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763549.559138:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2983 00000100:00080000:1.0:1789763549.559142:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763549.559146:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789763549.559155:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763549.559193:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024757120:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789763549.559219:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2d180 x1876700024757248/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763549.559260:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763549.559264:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763549.559266:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763549.559269:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2983 00000100:00080000:1.0:1789763549.559272:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763549.559276:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789763549.559280:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763549.559292:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2ed80 x1876700024757376/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763549.559361:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024757248:192.168.203.107@tcp:38:kworker.0 00000100:00100000:2.0:1789763549.559389:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024757376:192.168.203.107@tcp:38:kworker.0 00010000:00010000:2.0:1789763549.815079:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763550.070987:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763550.838978:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763551.094953:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789763551.863092:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763552.119004:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000400:00000100:1.0:1789763552.631158:0:2331:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.107:988 00000400:00000100:1.0:1789763552.631243:0:2331:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.107@tcp at host 192.168.203.107 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1789763552.631261:0:2331:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.107@tcp 00000800:00000100:1.0:1789763552.631297:0:2331:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.107@tcp 00000800:00000100:1.0:1789763552.631305:0:2331:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.107@tcp 00000100:00100000:2.0:1789763552.631403:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763549/real 1789763552] req@ffffa06c08e2d180 x1876700024757248/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763574 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763552.631489:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2d180 x1876700024757248/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763574 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763552.631512:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763552.631520:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789763552.631599:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024757248:192.168.203.107@tcp:38:kworker.0 00000100:00100000:2.0:1789763552.631619:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763549/real 1789763552] req@ffffa06c08e2ed80 x1876700024757376/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763574 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763552.631648:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2ed80 x1876700024757376/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763574 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763552.631678:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763552.631685:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789763552.631691:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024757376:192.168.203.107@tcp:38:kworker.0 00000100:00100000:2.0:1789763552.631705:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763549/real 1789763552] req@ffffa06c08e2ca80 x1876700024757120/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763574 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763552.631740:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2ca80 x1876700024757120/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763574 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763552.631753:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763552.631759:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789763552.631765:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024757120:192.168.203.107@tcp:250:kworker.0 00010000:00010000:2.0:1789763552.887073:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763553.143043:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763553.911116:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763554.166977:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:1.0:1789763554.679090:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763554.679106:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789763554.679112:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763554.679129:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2988 00000100:00080000:1.0:1789763554.679146:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763554.679180:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789763554.679247:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2c700 x1876700024757504/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763554.679293:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763554.679299:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763554.679303:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763554.679312:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2988 00000100:00080000:1.0:1789763554.679321:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763554.679330:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789763554.679346:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763554.679360:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024757504:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789763554.679379:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2c000 x1876700024757632/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763554.679432:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763554.679440:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763554.679445:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763554.679453:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2988 00000800:00000100:2.0:1789763554.679461:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00080000:1.0:1789763554.679468:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763554.679481:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789763554.679493:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763554.679522:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2e300 x1876700024757760/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763554.679545:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763554/real 1789763554] req@ffffa06c08e2c700 x1876700024757504/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763579 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763554.679584:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2c700 x1876700024757504/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763579 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763554.679605:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763554.679614:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789763554.679622:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024757504:192.168.203.107@tcp:250:kworker.0 00000100:00100000:2.0:1789763554.679661:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024757632:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789763554.679708:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789763554.679735:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763554/real 1789763554] req@ffffa06c08e2c000 x1876700024757632/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763579 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763554.679760:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2c000 x1876700024757632/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763579 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763554.679776:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763554.679783:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789763554.679790:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024757632:192.168.203.107@tcp:38:kworker.0 00000100:00100000:2.0:1789763554.679802:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024757760:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789763554.679895:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789763554.679925:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763554/real 1789763554] req@ffffa06c08e2e300 x1876700024757760/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763579 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763554.679981:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2e300 x1876700024757760/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763579 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763554.679999:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763554.680006:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789763554.680013:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024757760:192.168.203.107@tcp:38:kworker.0 00010000:00010000:1.0:1789763554.935117:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763555.190957:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763555.959047:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763556.214948:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763556.982993:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763557.238912:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789763558.007069:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763558.262983:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763559.031019:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763559.286954:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789763559.798998:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763559.799012:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763559.799017:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763559.799031:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2993 00000100:00080000:0.0:1789763559.799045:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763559.799054:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789763559.799109:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d2300 x1876700024757888/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763559.799179:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763559.799187:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763559.799212:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763559.799222:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2993 00000100:00080000:0.0:1789763559.799230:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763559.799237:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789763559.799251:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763559.799267:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024757888:192.168.203.107@tcp:250:kworker.0 00000100:00100000:0.0:1789763559.799277:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0000 x1876700024758016/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763559.799299:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763559.799304:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763559.799307:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763559.799312:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2993 00000100:00080000:0.0:1789763559.799316:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763559.799322:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789763559.799328:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763559.799344:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d3b80 x1876700024758144/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789763559.799381:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789763559.799454:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763559/real 1789763559] req@ffffa06c091d2300 x1876700024757888/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763584 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763559.799491:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d2300 x1876700024757888/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763584 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763559.799513:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763559.799521:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789763559.799528:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024757888:192.168.203.107@tcp:250:kworker.0 00000100:00100000:2.0:1789763559.799569:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024758016:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789763559.799635:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789763559.799656:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763559/real 1789763559] req@ffffa06c091d0000 x1876700024758016/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763584 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763559.799673:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0000 x1876700024758016/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763584 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763559.799714:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763559.799722:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789763559.799730:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024758016:192.168.203.107@tcp:38:kworker.0 00000100:00100000:2.0:1789763559.799742:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024758144:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789763559.799808:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789763559.799829:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763559/real 1789763559] req@ffffa06c091d3b80 x1876700024758144/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763584 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763559.799889:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d3b80 x1876700024758144/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763584 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763559.799901:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763559.799905:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789763559.799910:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024758144:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789763560.055057:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763560.310940:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789763561.079176:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763561.334958:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763562.103060:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763562.358990:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763563.127039:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763563.382967:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789763564.151132:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763564.406979:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789763564.919116:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763564.919150:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763564.919159:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763564.919179:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 2998 00000100:00080000:2.0:1789763564.919200:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763564.919214:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789763564.919286:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50000 x1876700024758272/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763564.919399:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763564.919408:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763564.919413:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763564.919423:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 2998 00000100:00080000:2.0:1789763564.919444:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763564.919453:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789763564.919470:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763564.919494:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024758272:192.168.203.107@tcp:250:kworker.0 00000100:00100000:2.0:1789763564.919497:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51500 x1876700024758400/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763564.919529:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763564.919534:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763564.919537:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763564.919541:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 2998 00000100:00080000:2.0:1789763564.919548:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763564.919552:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789763564.919559:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763564.919576:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50e00 x1876700024758528/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789763564.919630:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789763564.919698:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763564/real 1789763564] req@ffffa06c07e50000 x1876700024758272/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763589 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763564.919735:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50000 x1876700024758272/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763589 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763564.919758:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763564.919767:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789763564.919776:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024758272:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789763564.919822:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024758400:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789763564.919917:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789763564.919953:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763564/real 1789763564] req@ffffa06c07e51500 x1876700024758400/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763589 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763564.919971:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51500 x1876700024758400/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763589 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763564.919983:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763564.919987:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789763564.919992:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024758400:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789763564.919999:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024758528:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789763564.920028:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789763564.920041:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763564/real 1789763564] req@ffffa06c07e50e00 x1876700024758528/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763589 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763564.920055:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50e00 x1876700024758528/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763589 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763564.920065:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763564.920070:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789763564.920076:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024758528:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789763565.175047:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763565.430975:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763566.199029:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763566.454967:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789763567.223148:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763567.478936:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763568.246993:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763568.503009:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763569.271154:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763569.526955:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789763570.039096:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763570.039106:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763570.039112:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763570.039126:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 3003 00000100:00080000:0.0:1789763570.039150:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763570.039160:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789763570.039206:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d3b80 x1876700024758656/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763570.039266:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763570.039270:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763570.039273:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763570.039282:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 3003 00000100:00080000:0.0:1789763570.039291:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763570.039298:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789763570.039310:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763570.039332:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024758656:192.168.203.107@tcp:250:kworker.0 00000100:00100000:0.0:1789763570.039335:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0000 x1876700024758784/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763570.039359:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763570.039365:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763570.039369:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763570.039378:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 3003 00000100:00080000:0.0:1789763570.039388:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763570.039393:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789763570.039405:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763570.039431:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d2300 x1876700024758912/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789763570.039458:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789763570.039533:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763570/real 1789763570] req@ffffa06c091d3b80 x1876700024758656/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763595 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763570.039556:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d3b80 x1876700024758656/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763595 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763570.039579:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763570.039584:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789763570.039589:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024758656:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789763570.039618:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024758784:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789763570.039648:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789763570.039663:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763570/real 1789763570] req@ffffa06c091d0000 x1876700024758784/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763595 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763570.039677:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0000 x1876700024758784/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763595 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763570.039686:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763570.039690:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789763570.039693:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024758784:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789763570.039699:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024758912:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789763570.039716:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789763570.039726:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763570/real 1789763570] req@ffffa06c091d2300 x1876700024758912/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763595 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763570.039739:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d2300 x1876700024758912/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763595 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763570.039748:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763570.039752:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789763570.039756:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024758912:192.168.203.107@tcp:38:kworker.0 00010000:00010000:1.0:1789763570.295103:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763570.550946:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763571.319003:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763571.574938:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763572.343197:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763572.599051:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789763573.367099:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763573.622969:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763574.391072:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763574.646970:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789763575.158985:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763575.158996:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763575.159001:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763575.159027:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 3008 00000100:00080000:0.0:1789763575.159066:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763575.159073:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789763575.159139:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d1500 x1876700024759040/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763575.159176:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763575.159182:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763575.159185:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763575.159191:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 3008 00000100:00080000:0.0:1789763575.159197:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763575.159203:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789763575.159219:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763575.159239:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0e00 x1876700024759168/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763575.159249:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024759040:192.168.203.107@tcp:250:kworker.0 00000100:00080000:0.0:1789763575.159257:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763575.159262:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763575.159264:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763575.159269:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 3008 00000100:00080000:0.0:1789763575.159274:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763575.159279:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789763575.159286:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763575.159299:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d2a00 x1876700024759296/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789763575.159364:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789763575.159421:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763575/real 1789763575] req@ffffa06c091d1500 x1876700024759040/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763600 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763575.159443:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d1500 x1876700024759040/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763600 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763575.159453:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763575.159456:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789763575.159460:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024759040:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789763575.159487:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024759168:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789763575.159507:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789763575.159517:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763575/real 1789763575] req@ffffa06c091d0e00 x1876700024759168/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763600 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763575.159525:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0e00 x1876700024759168/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763600 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763575.159532:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763575.159534:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789763575.159536:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024759168:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789763575.159540:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024759296:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789763575.159550:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789763575.159566:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763575/real 1789763575] req@ffffa06c091d2a00 x1876700024759296/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763600 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763575.159574:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d2a00 x1876700024759296/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763600 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763575.159580:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763575.159582:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789763575.159584:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024759296:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789763575.415101:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763575.670966:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789763576.439132:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763576.694985:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763577.463060:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763577.719027:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763578.487081:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763578.742940:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789763579.511103:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763579.767003:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789763580.278956:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763580.278989:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763580.278996:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763580.279013:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 3014 00000100:00080000:0.0:1789763580.279031:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763580.279044:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789763580.279109:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d2680 x1876700024759424/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763580.279153:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763580.279159:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763580.279162:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763580.279167:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 3014 00000100:00080000:0.0:1789763580.279172:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763580.279176:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789763580.279192:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763580.279211:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0380 x1876700024759552/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763580.279227:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763580.279233:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763580.279236:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763580.279240:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 3014 00000100:00080000:0.0:1789763580.279244:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763580.279250:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789763580.279260:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024759424:192.168.203.107@tcp:250:kworker.0 00000008:00100000:0.0:1789763580.279262:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763580.279331:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d3100 x1876700024759680/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789763580.279370:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789763580.279431:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763580/real 1789763580] req@ffffa06c091d2680 x1876700024759424/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763605 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763580.279456:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d2680 x1876700024759424/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763605 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763580.279471:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763580.279477:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789763580.279483:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024759424:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789763580.279493:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024759552:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789763580.279517:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789763580.279532:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763580/real 1789763580] req@ffffa06c091d0380 x1876700024759552/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763605 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763580.279547:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0380 x1876700024759552/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763605 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763580.279578:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763580.279582:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789763580.279587:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024759552:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789763580.279622:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024759680:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789763580.279650:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789763580.279662:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763580/real 1789763580] req@ffffa06c091d3100 x1876700024759680/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763605 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763580.279677:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d3100 x1876700024759680/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763605 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763580.279686:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763580.279690:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789763580.279694:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024759680:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789763580.535084:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763580.791045:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789763581.559105:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763581.814999:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789763582.583086:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763582.839035:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763583.607045:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763583.863004:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789763584.631050:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763584.886997:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789763585.399003:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763585.399014:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763585.399019:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763585.399030:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 3019 00000100:00080000:0.0:1789763585.399042:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763585.399050:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789763585.399092:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d1f80 x1876700024759808/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763585.399124:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763585.399128:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763585.399130:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763585.399140:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 3019 00000100:00080000:0.0:1789763585.399144:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763585.399147:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789763585.399158:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763585.399171:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0a80 x1876700024759936/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763585.399185:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763585.399190:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763585.399193:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1789763585.399198:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024759808:192.168.203.107@tcp:250:kworker.0 00000100:00080000:0.0:1789763585.399200:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 3019 00000100:00080000:0.0:1789763585.399209:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763585.399233:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789763585.399239:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763585.399255:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0700 x1876700024760064/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789763585.399280:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789763585.399342:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763585/real 1789763585] req@ffffa06c091d1f80 x1876700024759808/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763610 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763585.399383:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d1f80 x1876700024759808/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763610 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763585.399402:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763585.399409:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789763585.399425:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024759808:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789763585.399453:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024759936:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789763585.399479:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789763585.399493:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763585/real 1789763585] req@ffffa06c091d0a80 x1876700024759936/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763610 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763585.399506:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0a80 x1876700024759936/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763610 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763585.399515:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763585.399519:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789763585.399522:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024759936:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789763585.399527:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024760064:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789763585.399548:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789763585.399556:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763585/real 1789763585] req@ffffa06c091d0700 x1876700024760064/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763610 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763585.399568:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0700 x1876700024760064/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763610 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763585.399575:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763585.399578:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789763585.399581:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024760064:192.168.203.107@tcp:38:kworker.0 00010000:00010000:0.0:1789763585.655148:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763585.911040:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763586.679107:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763586.934952:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763587.703069:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763587.958990:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763588.726976:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763588.982930:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763589.751031:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763590.006927:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789763590.519020:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763590.519050:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763590.519054:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763590.519066:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 3024 00000100:00080000:2.0:1789763590.519095:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763590.519107:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789763590.519160:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51f80 x1876700024760192/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763590.519208:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763590.519213:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763590.519216:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763590.519222:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 3024 00000100:00080000:2.0:1789763590.519228:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763590.519233:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789763590.519246:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763590.519267:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e53b80 x1876700024760320/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763590.519286:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024760192:192.168.203.107@tcp:250:kworker.0 00000100:00080000:2.0:1789763590.519290:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763590.519303:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763590.519307:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763590.519334:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 3024 00000100:00080000:2.0:1789763590.519343:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763590.519351:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789763590.519362:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763590.519409:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50380 x1876700024760448/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789763590.519444:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789763590.519523:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763590/real 1789763590] req@ffffa06c07e51f80 x1876700024760192/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763615 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763590.519555:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51f80 x1876700024760192/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763615 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763590.519569:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763590.519574:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789763590.519579:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024760192:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789763590.519628:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e53b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024760320:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789763590.519663:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789763590.519680:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763590/real 1789763590] req@ffffa06c07e53b80 x1876700024760320/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763615 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763590.519696:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e53b80 x1876700024760320/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763615 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763590.519708:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763590.519712:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789763590.519716:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e53b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024760320:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789763590.519723:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024760448:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789763590.519743:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789763590.519780:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763590/real 1789763590] req@ffffa06c07e50380 x1876700024760448/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763615 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763590.519793:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50380 x1876700024760448/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763615 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763590.519820:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763590.519824:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789763590.519828:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024760448:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789763590.775032:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763591.031003:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763591.799051:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763592.054951:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763592.823009:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763593.078963:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763593.847095:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763594.102979:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763594.871291:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763595.127006:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:3.0:1789763595.639048:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763595.639061:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763595.639066:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763595.639078:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 3029 00000100:00080000:3.0:1789763595.639091:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763595.639100:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:3.0:1789763595.639145:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872c700 x1876700024760576/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763595.639204:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763595.639209:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763595.639212:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763595.639217:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 3029 00000100:00080000:3.0:1789763595.639222:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763595.639227:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789763595.639239:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763595.639257:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872e680 x1876700024760704/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763595.639275:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1789763595.639276:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024760576:192.168.203.107@tcp:250:kworker.0 00000100:00080000:3.0:1789763595.639286:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763595.639294:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763595.639302:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 3029 00000100:00080000:3.0:1789763595.639309:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763595.639318:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789763595.639328:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763595.639353:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872dc00 x1876700024760832/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789763595.639379:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789763595.639443:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763595/real 1789763595] req@ffffa06c1872c700 x1876700024760576/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763620 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763595.639468:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872c700 x1876700024760576/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763620 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763595.639481:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763595.639486:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789763595.639491:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024760576:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789763595.639521:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024760704:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789763595.639551:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789763595.639565:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763595/real 1789763595] req@ffffa06c1872e680 x1876700024760704/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763620 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763595.639580:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872e680 x1876700024760704/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763620 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763595.639590:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763595.639594:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789763595.639598:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024760704:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789763595.639631:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024760832:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789763595.639660:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789763595.639674:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763595/real 1789763595] req@ffffa06c1872dc00 x1876700024760832/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763620 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763595.639688:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872dc00 x1876700024760832/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763620 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763595.639697:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763595.639701:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789763595.639705:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024760832:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789763595.895063:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763596.151050:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763596.919174:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763597.175034:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789763597.943074:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763598.199008:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763598.967102:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763599.222977:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763599.990999:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763600.246960:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789763600.759021:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763600.759042:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763600.759047:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763600.759061:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 3034 00000100:00080000:0.0:1789763600.759075:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763600.759084:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789763600.759137:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0700 x1876700024760960/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763600.759176:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763600.759181:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763600.759184:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763600.759189:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 3034 00000100:00080000:0.0:1789763600.759193:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763600.759197:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789763600.759209:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763600.759227:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0a80 x1876700024761088/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763600.759243:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763600.759248:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:1.0:1789763600.759265:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024760960:192.168.203.107@tcp:250:kworker.0 00000100:00080000:0.0:1789763600.759274:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763600.759285:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 3034 00000100:00080000:0.0:1789763600.759290:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763600.759296:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789763600.759304:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763600.759324:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d1f80 x1876700024761216/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789763600.759361:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789763600.759417:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763600/real 1789763600] req@ffffa06c091d0700 x1876700024760960/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763625 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763600.759440:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0700 x1876700024760960/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763625 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763600.759454:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763600.759459:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789763600.759464:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024760960:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789763600.759474:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024761088:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789763600.759498:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789763600.759512:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763600/real 1789763600] req@ffffa06c091d0a80 x1876700024761088/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763625 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763600.759541:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0a80 x1876700024761088/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763625 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763600.759553:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763600.759557:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789763600.759561:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024761088:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789763600.759593:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024761216:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789763600.759617:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789763600.759629:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763600/real 1789763600] req@ffffa06c091d1f80 x1876700024761216/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763625 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763600.759643:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d1f80 x1876700024761216/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763625 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763600.759653:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763600.759658:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789763600.759662:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024761216:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789763601.015046:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763601.270990:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763602.039026:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763602.294952:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763603.063014:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763603.318969:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763604.087083:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763604.342993:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763605.111088:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763605.366987:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:1.0:1789763605.879030:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763605.879042:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789763605.879048:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763605.879062:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 3039 00000100:00080000:1.0:1789763605.879077:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763605.879088:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789763605.879139:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2d500 x1876700024761344/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763605.879204:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763605.879209:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763605.879212:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763605.879219:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 3039 00000100:00080000:1.0:1789763605.879226:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763605.879235:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789763605.879249:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763605.879266:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024761344:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789763605.879275:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2e300 x1876700024761472/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763605.879294:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763605.879299:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763605.879302:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763605.879306:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 3039 00000100:00080000:1.0:1789763605.879311:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763605.879315:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789763605.879324:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763605.879349:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2c000 x1876700024761600/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789763605.879411:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789763605.879494:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763605/real 1789763605] req@ffffa06c08e2d500 x1876700024761344/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763630 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763605.879536:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2d500 x1876700024761344/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763630 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763605.879567:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763605.879575:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789763605.879592:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024761344:192.168.203.107@tcp:250:kworker.0 00000100:00100000:0.0:1789763605.879656:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024761472:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789763605.879695:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789763605.879730:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763605/real 1789763605] req@ffffa06c08e2e300 x1876700024761472/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763630 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763605.879758:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2e300 x1876700024761472/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763630 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763605.879823:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763605.879828:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789763605.879833:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024761472:192.168.203.107@tcp:38:kworker.0 00000100:00100000:0.0:1789763605.879881:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024761600:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789763605.879927:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789763605.879965:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763605/real 1789763605] req@ffffa06c08e2c000 x1876700024761600/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763630 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763605.879981:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2c000 x1876700024761600/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763630 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763605.879991:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763605.879995:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789763605.879999:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024761600:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789763606.135041:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763606.390966:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763607.159097:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763607.415000:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789763608.183012:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763608.438941:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763609.207004:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763609.462971:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763610.231037:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763610.487006:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:1.0:1789763610.998963:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763610.998981:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789763610.998989:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763610.999004:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 3044 00000100:00080000:1.0:1789763610.999037:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763610.999048:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789763610.999093:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2c700 x1876700024761728/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763610.999132:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763610.999138:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763610.999142:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763610.999149:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 3044 00000100:00080000:1.0:1789763610.999156:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763610.999167:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789763610.999184:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763610.999198:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024761728:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789763610.999215:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2ca80 x1876700024761856/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763610.999245:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763610.999252:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763610.999255:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763610.999261:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 3044 00000100:00080000:1.0:1789763610.999288:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763610.999295:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000800:00000100:0.0:1789763610.999303:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000008:00100000:1.0:1789763610.999307:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763610.999332:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2ed80 x1876700024761984/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789763610.999365:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763610/real 1789763610] req@ffffa06c08e2c700 x1876700024761728/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763635 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763610.999403:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2c700 x1876700024761728/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763635 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763610.999417:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763610.999423:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789763610.999445:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024761728:192.168.203.107@tcp:250:kworker.0 00000100:00100000:0.0:1789763610.999485:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024761856:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789763610.999534:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789763610.999555:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763610/real 1789763610] req@ffffa06c08e2ca80 x1876700024761856/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763635 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763610.999575:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2ca80 x1876700024761856/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763635 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763610.999593:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763610.999600:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789763610.999607:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024761856:192.168.203.107@tcp:38:kworker.0 00000100:00100000:0.0:1789763610.999621:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024761984:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789763610.999664:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789763610.999688:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763610/real 1789763610] req@ffffa06c08e2ed80 x1876700024761984/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763635 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763610.999714:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2ed80 x1876700024761984/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763635 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763610.999730:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763610.999735:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789763610.999739:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024761984:192.168.203.107@tcp:38:kworker.0 00010000:00010000:0.0:1789763611.255040:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763611.510992:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763612.279075:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763612.534959:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763613.302998:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763613.558973:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789763614.327130:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763614.582955:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789763615.351056:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763615.606983:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:3.0:1789763616.118979:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763616.118995:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763616.119020:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763616.119037:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 3049 00000100:00080000:3.0:1789763616.119068:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763616.119096:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:3.0:1789763616.119174:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872ed80 x1876700024762112/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763616.119219:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763616.119230:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763616.119233:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763616.119239:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 3049 00000100:00080000:3.0:1789763616.119246:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763616.119253:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789763616.119270:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763616.119298:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024762112:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789763616.119308:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872ea00 x1876700024762240/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763616.119329:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763616.119335:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763616.119338:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763616.119343:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 3049 00000100:00080000:3.0:1789763616.119349:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763616.119354:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789763616.119360:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763616.119376:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872f480 x1876700024762368/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789763616.119492:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024762240:192.168.203.107@tcp:38:kworker.0 00000100:00100000:0.0:1789763616.119560:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024762368:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789763616.375081:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763616.630950:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789763617.398954:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763617.654986:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789763618.423079:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763618.678973:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000400:00000100:2.0:1789763619.191307:0:2328:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.107:988 00000400:00000100:2.0:1789763619.191383:0:2328:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.107@tcp at host 192.168.203.107 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1789763619.191412:0:2328:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.107@tcp 00000800:00000100:2.0:1789763619.191474:0:2328:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.107@tcp 00000800:00000100:2.0:1789763619.191494:0:2328:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.107@tcp 00000100:00100000:0.0:1789763619.191573:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763616/real 1789763619] req@ffffa06c1872ea00 x1876700024762240/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763641 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763619.191622:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872ea00 x1876700024762240/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763641 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763619.191646:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763619.191656:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789763619.191665:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024762240:192.168.203.107@tcp:38:kworker.0 00000100:00100000:0.0:1789763619.191714:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763616/real 1789763619] req@ffffa06c1872f480 x1876700024762368/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763641 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763619.191743:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872f480 x1876700024762368/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763641 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763619.191792:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763619.191801:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789763619.191808:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024762368:192.168.203.107@tcp:38:kworker.0 00000100:00100000:0.0:1789763619.191823:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763616/real 1789763619] req@ffffa06c1872ed80 x1876700024762112/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763641 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763619.191928:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872ed80 x1876700024762112/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763641 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763619.191965:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763619.191974:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789763619.191981:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024762112:192.168.203.107@tcp:250:kworker.0 00010000:00010000:0.0:1789763619.447083:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763619.702960:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789763620.471040:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763620.726948:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789763621.239065:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763621.239089:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763621.239095:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763621.239112:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 3055 00000100:00080000:2.0:1789763621.239129:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763621.239142:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789763621.239199:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e52680 x1876700024762496/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763621.239253:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763621.239261:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763621.239265:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763621.239291:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 3055 00000100:00080000:2.0:1789763621.239300:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763621.239310:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789763621.239329:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763621.239347:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e52680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024762496:192.168.203.107@tcp:250:kworker.0 00000100:00100000:2.0:1789763621.239380:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e53800 x1876700024762624/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763621.239410:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763621.239417:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763621.239421:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763621.239429:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 3055 00000100:00080000:2.0:1789763621.239440:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1789763621.239441:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00080000:2.0:1789763621.239458:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789763621.239493:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763621.239531:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51c00 x1876700024762752/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789763621.239555:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763621/real 1789763621] req@ffffa06c07e52680 x1876700024762496/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763646 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763621.239601:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e52680 x1876700024762496/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763646 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763621.239624:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763621.239633:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789763621.239641:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e52680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024762496:192.168.203.107@tcp:250:kworker.0 00000100:00100000:0.0:1789763621.239704:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e53800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024762624:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789763621.239754:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789763621.239784:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763621/real 1789763621] req@ffffa06c07e53800 x1876700024762624/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763646 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763621.239811:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e53800 x1876700024762624/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763646 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763621.239830:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763621.239837:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789763621.239944:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e53800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024762624:192.168.203.107@tcp:38:kworker.0 00000100:00100000:0.0:1789763621.239959:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024762752:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789763621.240004:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789763621.240038:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763621/real 1789763621] req@ffffa06c07e51c00 x1876700024762752/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763646 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763621.240065:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51c00 x1876700024762752/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763646 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763621.240083:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763621.240091:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789763621.240099:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024762752:192.168.203.107@tcp:38:kworker.0 00010000:00010000:2.0:1789763621.495053:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763621.750945:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789763622.519028:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763622.774980:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789763623.543027:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763623.798956:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789763624.567057:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763624.822938:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789763625.591130:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763625.846940:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789763626.358997:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763626.359010:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763626.359015:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763626.359029:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 3060 00000100:00080000:0.0:1789763626.359043:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763626.359052:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789763626.359100:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d3100 x1876700024762880/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763626.359143:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763626.359148:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763626.359162:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763626.359189:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 3060 00000100:00080000:0.0:1789763626.359199:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763626.359209:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789763626.359243:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763626.359246:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024762880:192.168.203.107@tcp:250:kworker.0 00000100:00100000:0.0:1789763626.359272:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0380 x1876700024763008/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763626.359317:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763626.359325:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763626.359329:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763626.359338:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 3060 00000100:00080000:0.0:1789763626.359346:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1789763626.359348:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00080000:0.0:1789763626.359363:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789763626.359374:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763626.359397:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d2680 x1876700024763136/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763626.359414:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763626/real 1789763626] req@ffffa06c091d3100 x1876700024762880/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763651 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763626.359459:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d3100 x1876700024762880/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763651 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763626.359474:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763626.359479:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789763626.359484:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024762880:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789763626.359511:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024763008:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789763626.359557:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789763626.359573:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763626/real 1789763626] req@ffffa06c091d0380 x1876700024763008/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763651 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763626.359588:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0380 x1876700024763008/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763651 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763626.359600:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763626.359604:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789763626.359608:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024763008:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789763626.359616:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024763136:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789763626.359669:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789763626.359686:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763626/real 1789763626] req@ffffa06c091d2680 x1876700024763136/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763651 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763626.359725:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d2680 x1876700024763136/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763651 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763626.359736:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763626.359740:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789763626.359744:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024763136:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789763626.615011:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763626.870980:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789763627.639059:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763627.894946:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789763628.663119:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763628.919013:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763629.687121:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763629.943019:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789763630.711108:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763630.966990:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789763631.479036:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763631.479053:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763631.479059:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763631.479076:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 3065 00000100:00080000:0.0:1789763631.479093:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763631.479105:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789763631.479169:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d2a00 x1876700024763264/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763631.479241:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763631.479246:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763631.479249:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763631.479255:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 3065 00000100:00080000:0.0:1789763631.479260:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763631.479265:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789763631.479280:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763631.479298:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0e00 x1876700024763392/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763631.479314:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763631.479321:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763631.479324:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763631.479330:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 3065 00000100:00080000:0.0:1789763631.479339:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763631.479345:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789763631.479352:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763631.479366:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d1500 x1876700024763520/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763631.479387:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024763264:192.168.203.107@tcp:250:kworker.0 00000800:00000100:1.0:1789763631.479486:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789763631.479541:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763631/real 1789763631] req@ffffa06c091d2a00 x1876700024763264/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763656 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763631.479566:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d2a00 x1876700024763264/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763656 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763631.479582:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763631.479587:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789763631.479593:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024763264:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789763631.479648:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024763392:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789763631.479685:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789763631.479702:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763631/real 1789763631] req@ffffa06c091d0e00 x1876700024763392/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763656 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763631.479718:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0e00 x1876700024763392/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763656 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763631.479755:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763631.479760:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789763631.479764:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024763392:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789763631.479800:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024763520:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789763631.479828:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789763631.479841:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763631/real 1789763631] req@ffffa06c091d1500 x1876700024763520/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763656 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763631.479907:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d1500 x1876700024763520/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763656 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763631.479917:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763631.479922:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789763631.479926:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024763520:192.168.203.107@tcp:38:kworker.0 00010000:00010000:2.0:1789763631.735053:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763631.990979:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763632.759015:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763633.014980:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763633.783055:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763634.038981:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789763634.807015:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763635.062971:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789763635.831059:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763636.086986:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:1.0:1789763636.599003:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763636.599037:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789763636.599044:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763636.599060:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 3070 00000100:00080000:1.0:1789763636.599077:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763636.599087:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789763636.599135:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2d180 x1876700024763648/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763636.599174:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763636.599194:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763636.599198:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763636.599206:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 3070 00000100:00080000:1.0:1789763636.599212:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763636.599219:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789763636.599231:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763636.599236:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024763648:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789763636.599255:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2ed80 x1876700024763776/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763636.599273:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763636.599279:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763636.599281:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763636.599286:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 3070 00000100:00080000:1.0:1789763636.599312:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1789763636.599321:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00080000:1.0:1789763636.599324:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789763636.599339:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763636.599366:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2ca80 x1876700024763904/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789763636.599386:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763636/real 1789763636] req@ffffa06c08e2d180 x1876700024763648/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763661 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763636.599450:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2d180 x1876700024763648/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763661 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763636.599464:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763636.599469:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789763636.599474:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024763648:192.168.203.107@tcp:250:kworker.0 00000100:00100000:0.0:1789763636.599505:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024763776:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789763636.599536:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789763636.599551:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763636/real 1789763636] req@ffffa06c08e2ed80 x1876700024763776/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763661 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763636.599595:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2ed80 x1876700024763776/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763661 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763636.599607:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763636.599612:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789763636.599619:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024763776:192.168.203.107@tcp:38:kworker.0 00000100:00100000:0.0:1789763636.599630:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024763904:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789763636.599669:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789763636.599690:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763636/real 1789763636] req@ffffa06c08e2ca80 x1876700024763904/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763661 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763636.599713:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2ca80 x1876700024763904/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763661 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763636.599740:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763636.599744:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789763636.599748:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024763904:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789763636.855066:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789763637.111013:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789763637.879176:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763638.135058:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789763638.903123:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763639.159001:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763639.927038:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763640.182986:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789763640.951249:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763641.206962:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789763641.719039:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763641.719055:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763641.719063:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763641.719080:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 3075 00000100:00080000:0.0:1789763641.719096:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763641.719106:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789763641.719153:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d2300 x1876700024764032/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763641.719208:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763641.719218:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763641.719223:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763641.719233:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 3075 00000100:00080000:0.0:1789763641.719244:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763641.719253:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789763641.719271:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763641.719302:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0000 x1876700024764160/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763641.719329:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1789763641.719350:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024764032:192.168.203.107@tcp:250:kworker.0 00000100:00080000:0.0:1789763641.719365:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763641.719371:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763641.719379:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 3075 00000100:00080000:0.0:1789763641.719386:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763641.719394:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789763641.719404:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763641.719425:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d3b80 x1876700024764288/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789763641.719587:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763641.719640:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763641/real 1789763641] req@ffffa06c091d2300 x1876700024764032/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763666 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763641.719664:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d2300 x1876700024764032/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763666 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763641.719678:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763641.719683:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763641.719689:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024764032:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789763641.719699:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024764160:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763641.719726:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763641.719991:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763641/real 1789763641] req@ffffa06c091d0000 x1876700024764160/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763666 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763641.720016:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0000 x1876700024764160/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763666 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763641.720029:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763641.720035:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763641.720040:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024764160:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789763641.720179:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024764288:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763641.720219:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763641.720236:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763641/real 1789763641] req@ffffa06c091d3b80 x1876700024764288/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763666 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763641.720274:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d3b80 x1876700024764288/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763666 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763641.720285:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763641.720289:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763641.720293:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024764288:192.168.203.107@tcp:38:kworker.0 00010000:00010000:2.0:1789763641.975043:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763642.230977:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763642.999114:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763643.255008:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789763644.023024:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763644.278949:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789763645.047073:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763645.302977:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763646.071066:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763646.326950:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789763646.838951:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763646.838975:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763646.838980:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763646.838991:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 3080 00000100:00080000:0.0:1789763646.839004:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763646.839012:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789763646.839063:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d3b80 x1876700024764416/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763646.839094:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763646.839099:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763646.839101:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763646.839105:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 3080 00000100:00080000:0.0:1789763646.839109:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763646.839113:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789763646.839123:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763646.839135:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0000 x1876700024764544/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763646.839163:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763646.839167:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763646.839170:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763646.839176:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 3080 00000100:00080000:0.0:1789763646.839180:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763646.839185:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789763646.839191:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763646.839204:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d2300 x1876700024764672/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763646.839214:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024764416:192.168.203.107@tcp:250:kworker.0 00000800:00000100:3.0:1789763646.839293:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763646.839358:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763646/real 1789763646] req@ffffa06c091d3b80 x1876700024764416/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763671 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763646.839382:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d3b80 x1876700024764416/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763671 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763646.839397:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763646.839402:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763646.839408:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024764416:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789763646.839417:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024764544:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763646.839461:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763646.839478:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763646/real 1789763646] req@ffffa06c091d0000 x1876700024764544/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763671 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763646.839536:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0000 x1876700024764544/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763671 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763646.839546:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763646.839549:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763646.839553:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024764544:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789763646.839579:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024764672:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763646.839624:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763646.839638:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763646/real 1789763646] req@ffffa06c091d2300 x1876700024764672/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763671 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763646.839652:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d2300 x1876700024764672/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763671 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763646.839662:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763646.839665:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763646.839669:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024764672:192.168.203.107@tcp:38:kworker.0 00010000:00010000:0.0:1789763647.095049:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763647.350940:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763648.119014:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763648.374946:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763649.142944:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763649.398945:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789763650.167020:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789763650.422999:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763651.191027:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789763651.447020:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789763651.959087:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763651.959097:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763651.959102:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763651.959112:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 3085 00000100:00080000:2.0:1789763651.959148:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763651.959159:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789763651.959206:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e53480 x1876700024764800/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763651.959240:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763651.959246:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763651.959249:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763651.959256:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 3085 00000100:00080000:2.0:1789763651.959262:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763651.959268:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789763651.959282:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763651.959304:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50a80 x1876700024764928/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763651.959322:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763651.959326:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763651.959328:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763651.959333:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 3085 00000100:00080000:2.0:1789763651.959354:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763651.959362:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:3.0:1789763651.959376:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e53480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024764800:192.168.203.107@tcp:250:kworker.0 00000008:00100000:2.0:1789763651.959378:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763651.959415:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50700 x1876700024765056/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789763651.959565:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763651.959642:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763651/real 1789763651] req@ffffa06c07e53480 x1876700024764800/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763676 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763651.959670:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e53480 x1876700024764800/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763676 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763651.959686:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763651.959693:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763651.959699:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e53480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024764800:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789763651.959711:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024764928:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763651.959758:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763651.959784:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763651/real 1789763651] req@ffffa06c07e50a80 x1876700024764928/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763676 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763651.959810:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50a80 x1876700024764928/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763676 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763651.959828:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763651.959836:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763651.959843:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024764928:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789763651.959958:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024765056:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763651.959992:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763651.960009:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763651/real 1789763651] req@ffffa06c07e50700 x1876700024765056/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763676 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763651.960061:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50700 x1876700024765056/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763676 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763651.960073:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763651.960077:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763651.960081:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024765056:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789763652.215075:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789763652.470997:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789763653.239106:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789763653.494986:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763654.263157:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789763654.518918:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763655.286977:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789763655.542945:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789763656.311040:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789763656.566969:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:1.0:1789763657.078971:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763657.078992:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789763657.078997:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763657.079007:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 3090 00000100:00080000:1.0:1789763657.079021:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763657.079049:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789763657.079087:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2c700 x1876700024765184/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763657.079119:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763657.079123:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763657.079125:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763657.079136:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 3090 00000100:00080000:1.0:1789763657.079141:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763657.079145:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789763657.079155:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763657.079171:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2c000 x1876700024765312/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763657.079185:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763657.079189:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763657.079191:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763657.079196:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 3090 00000100:00080000:1.0:1789763657.079199:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763657.079202:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789763657.079207:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763657.079213:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024765184:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789763657.079221:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2e300 x1876700024765440/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789763657.079314:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763657.079376:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763657/real 1789763657] req@ffffa06c08e2c700 x1876700024765184/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763682 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763657.079405:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2c700 x1876700024765184/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763682 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763657.079442:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763657.079449:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763657.079456:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024765184:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789763657.079469:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024765312:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763657.079509:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763657.079534:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763657/real 1789763657] req@ffffa06c08e2c000 x1876700024765312/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763682 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763657.079582:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2c000 x1876700024765312/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763682 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763657.079598:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763657.079603:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763657.079609:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024765312:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789763657.079642:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024765440:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763657.079680:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763657.079700:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763657/real 1789763657] req@ffffa06c08e2e300 x1876700024765440/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763682 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763657.079721:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2e300 x1876700024765440/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763682 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763657.079734:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763657.079740:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763657.079745:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024765440:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789763657.334991:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789763657.590980:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763658.359159:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763658.615098:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789763659.383086:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763659.638943:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763660.407023:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763660.662984:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763661.431091:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763661.686955:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789763662.199028:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763662.199044:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763662.199050:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763662.199064:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 3096 00000100:00080000:0.0:1789763662.199081:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763662.199093:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789763662.199166:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d1500 x1876700024765568/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763662.199213:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763662.199220:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763662.199224:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763662.199231:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 3096 00000100:00080000:0.0:1789763662.199239:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763662.199265:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789763662.199280:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763662.199309:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0e00 x1876700024765696/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763662.199338:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1789763662.199346:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024765568:192.168.203.107@tcp:250:kworker.0 00000100:00080000:0.0:1789763662.199349:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763662.199354:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763662.199366:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 3096 00000100:00080000:0.0:1789763662.199422:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763662.199431:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789763662.199442:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763662.199468:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d2a00 x1876700024765824/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789763662.199491:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763662.199544:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763662/real 1789763662] req@ffffa06c091d1500 x1876700024765568/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763687 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763662.199565:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d1500 x1876700024765568/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763687 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763662.199578:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763662.199582:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763662.199587:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024765568:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789763662.199613:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024765696:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763662.199634:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763662.199648:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763662/real 1789763662] req@ffffa06c091d0e00 x1876700024765696/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763687 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763662.199666:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0e00 x1876700024765696/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763687 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763662.199680:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763662.199687:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763662.199693:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024765696:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789763662.199703:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024765824:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763662.199751:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763662.199772:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763662/real 1789763662] req@ffffa06c091d2a00 x1876700024765824/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763687 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763662.199813:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d2a00 x1876700024765824/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763687 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763662.199822:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763662.199826:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763662.199830:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024765824:192.168.203.107@tcp:38:kworker.0 00010000:00010000:0.0:1789763662.455010:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763662.710989:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763663.478975:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763663.734963:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763664.503073:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763664.759006:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789763665.527032:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763665.782997:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763666.551058:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763666.806922:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789763667.318945:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763667.318956:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763667.318978:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763667.318989:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 3101 00000100:00080000:0.0:1789763667.319001:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763667.319008:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789763667.319046:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d2680 x1876700024765952/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763667.319078:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763667.319082:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763667.319084:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763667.319089:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 3101 00000100:00080000:0.0:1789763667.319092:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763667.319096:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789763667.319106:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763667.319117:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0380 x1876700024766080/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763667.319129:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763667.319133:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763667.319135:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763667.319138:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 3101 00000100:00080000:0.0:1789763667.319141:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763667.319145:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789763667.319149:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763667.319159:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d3100 x1876700024766208/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763667.319208:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024765952:192.168.203.107@tcp:250:kworker.0 00000800:00000100:3.0:1789763667.319294:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763667.319392:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763667/real 1789763667] req@ffffa06c091d2680 x1876700024765952/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763692 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763667.319421:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d2680 x1876700024765952/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763692 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763667.319438:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763667.319445:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763667.319452:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024765952:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789763667.319464:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024766080:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763667.319504:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763667.319520:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763667/real 1789763667] req@ffffa06c091d0380 x1876700024766080/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763692 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763667.319533:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0380 x1876700024766080/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763692 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763667.319542:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763667.319545:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763667.319548:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024766080:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789763667.319587:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024766208:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763667.319607:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763667.319619:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763667/real 1789763667] req@ffffa06c091d3100 x1876700024766208/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763692 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763667.319630:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d3100 x1876700024766208/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763692 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763667.319638:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763667.319642:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763667.319645:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024766208:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789763667.575076:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763667.831030:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763668.599120:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763668.854950:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763669.623019:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763669.878942:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763670.647039:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763670.902996:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763671.671055:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763671.926963:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:1.0:1789763672.439044:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763672.439056:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789763672.439061:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763672.439074:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 3106 00000100:00080000:1.0:1789763672.439088:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763672.439098:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789763672.439146:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2d500 x1876700024766336/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763672.439183:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763672.439188:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763672.439191:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763672.439196:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 3106 00000100:00080000:1.0:1789763672.439200:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763672.439205:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789763672.439217:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763672.439235:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2e300 x1876700024766464/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763672.439252:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763672.439258:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:3.0:1789763672.439264:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024766336:192.168.203.107@tcp:250:kworker.0 00000100:00080000:1.0:1789763672.439265:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763672.439279:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 3106 00000100:00080000:1.0:1789763672.439286:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763672.439290:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789763672.439325:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763672.439366:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2c000 x1876700024766592/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789763672.439403:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763672.439470:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763672/real 1789763672] req@ffffa06c08e2d500 x1876700024766336/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763697 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763672.439494:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2d500 x1876700024766336/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763697 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763672.439508:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763672.439513:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763672.439518:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024766336:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789763672.439548:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024766464:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763672.439578:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763672.439593:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763672/real 1789763672] req@ffffa06c08e2e300 x1876700024766464/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763697 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763672.439608:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2e300 x1876700024766464/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763697 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763672.439619:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763672.439624:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763672.439628:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024766464:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789763672.439635:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024766592:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763672.439674:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763672.439687:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763672/real 1789763672] req@ffffa06c08e2c000 x1876700024766592/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763697 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763672.439702:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2c000 x1876700024766592/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763697 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763672.439712:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763672.439716:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763672.439720:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024766592:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789763672.695033:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763672.951005:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763673.719013:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763673.974918:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763674.743151:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763674.999021:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763675.766993:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763676.023012:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763676.791107:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763677.047007:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789763677.558957:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763677.558979:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763677.558983:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763677.558994:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 3111 00000100:00080000:2.0:1789763677.559006:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763677.559014:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789763677.559061:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e52d80 x1876700024766720/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763677.559084:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763677.559088:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763677.559091:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763677.559096:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 3111 00000100:00080000:2.0:1789763677.559100:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763677.559104:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789763677.559116:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763677.559117:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e52d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024766720:192.168.203.107@tcp:250:kworker.0 00000100:00100000:2.0:1789763677.559132:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e53100 x1876700024766848/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763677.559143:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763677.559146:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763677.559148:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763677.559151:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 3111 00000100:00080000:2.0:1789763677.559155:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763677.559158:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789763677.559161:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763677.559171:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e52a00 x1876700024766976/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789763677.559209:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763677.559337:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763677/real 1789763677] req@ffffa06c07e52d80 x1876700024766720/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763702 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763677.559495:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e52d80 x1876700024766720/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763702 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763677.559517:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763677.559566:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763677.559572:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e52d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024766720:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789763677.559601:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e53100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024766848:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763677.559623:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763677.559634:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763677/real 1789763677] req@ffffa06c07e53100 x1876700024766848/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763702 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763677.559643:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e53100 x1876700024766848/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763702 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763677.559649:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763677.559652:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763677.559654:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e53100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024766848:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789763677.559658:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e52a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024766976:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763677.559680:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763677.559687:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763677/real 1789763677] req@ffffa06c07e52a00 x1876700024766976/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763702 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763677.559695:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e52a00 x1876700024766976/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763702 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763677.559700:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763677.559701:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763677.559703:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e52a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024766976:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789763677.814971:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763678.070939:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763678.839050:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763679.094979:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789763679.863094:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763680.118961:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763680.887083:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763681.142974:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763681.911056:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763682.167023:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789763682.678973:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763682.678982:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763682.678994:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763682.679006:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 3116 00000100:00080000:2.0:1789763682.679034:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763682.679040:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789763682.679080:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e52a00 x1876700024767104/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763682.679117:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763682.679121:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763682.679123:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763682.679126:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 3116 00000100:00080000:2.0:1789763682.679129:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763682.679132:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789763682.679141:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763682.679157:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e53100 x1876700024767232/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763682.679170:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763682.679173:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763682.679175:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763682.679178:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 3116 00000100:00080000:2.0:1789763682.679180:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1789763682.679202:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e52a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024767104:192.168.203.107@tcp:250:kworker.0 00000100:00080000:2.0:1789763682.679203:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789763682.679214:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763682.679284:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e52d80 x1876700024767360/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763682.679458:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e53100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024767232:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789763682.679517:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e52d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024767360:192.168.203.107@tcp:38:kworker.0 00010000:00010000:0.0:1789763682.934998:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763683.190986:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763683.959021:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763684.215071:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763684.983004:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763685.238949:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000400:00000100:0.0:1789763685.751223:0:2329:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.107:988 00000400:00000100:0.0:1789763685.751307:0:2329:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.107@tcp at host 192.168.203.107 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1789763685.751335:0:2329:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.107@tcp 00000800:00000100:0.0:1789763685.751398:0:2329:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.107@tcp 00000800:00000100:0.0:1789763685.751416:0:2329:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.107@tcp 00000100:00100000:3.0:1789763685.751473:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763682/real 1789763685] req@ffffa06c07e52d80 x1876700024767360/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763707 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763685.751509:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e52d80 x1876700024767360/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763707 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763685.751532:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763685.751541:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763685.751550:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e52d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024767360:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789763685.751565:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763682/real 1789763685] req@ffffa06c07e52a00 x1876700024767104/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763707 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763685.751625:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e52a00 x1876700024767104/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763707 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763685.751643:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763685.751651:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763685.751658:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e52a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024767104:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789763685.751671:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763682/real 1789763685] req@ffffa06c07e53100 x1876700024767232/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763707 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763685.751696:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e53100 x1876700024767232/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763707 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763685.751730:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763685.751739:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763685.751747:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e53100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024767232:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789763686.007104:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763686.262956:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763687.031019:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763687.286975:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789763687.799105:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763687.799123:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763687.799131:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763687.799145:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 3121 00000100:00080000:0.0:1789763687.799159:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763687.799168:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789763687.799247:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d1f80 x1876700024767488/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763687.799300:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763687.799308:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763687.799312:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763687.799330:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 3121 00000100:00080000:0.0:1789763687.799340:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763687.799350:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:3.0:1789763687.799362:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024767488:192.168.203.107@tcp:250:kworker.0 00000008:00100000:0.0:1789763687.799371:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763687.799408:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0a80 x1876700024767616/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763687.799450:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763687.799455:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763687.799462:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:3.0:1789763687.799465:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00080000:0.0:1789763687.799472:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 3121 00000100:00080000:0.0:1789763687.799482:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763687.799493:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789763687.799505:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763687.799537:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0700 x1876700024767744/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763687.799548:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763687/real 1789763687] req@ffffa06c091d1f80 x1876700024767488/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763712 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763687.799604:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d1f80 x1876700024767488/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763712 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763687.799653:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763687.799664:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763687.799674:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024767488:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789763687.799720:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024767616:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763687.799774:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763687.799803:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763687/real 1789763687] req@ffffa06c091d0a80 x1876700024767616/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763712 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763687.799910:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0a80 x1876700024767616/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763712 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763687.799929:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763687.799937:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763687.799945:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024767616:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789763687.799960:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024767744:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763687.800035:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763687.800063:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763687/real 1789763687] req@ffffa06c091d0700 x1876700024767744/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763712 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763687.800090:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0700 x1876700024767744/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763712 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763687.800108:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763687.800116:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763687.800123:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024767744:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789763688.055120:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763688.311058:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763689.078989:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763689.334996:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763690.103021:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763690.358976:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763691.127066:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763691.382959:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763692.151014:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763692.406939:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789763692.918985:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763692.919002:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763692.919009:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763692.919023:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 3126 00000100:00080000:0.0:1789763692.919037:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763692.919046:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789763692.919096:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0700 x1876700024767872/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763692.919137:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763692.919165:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763692.919168:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763692.919175:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 3126 00000100:00080000:0.0:1789763692.919180:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763692.919185:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789763692.919228:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763692.919250:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0a80 x1876700024768000/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763692.919270:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024767872:192.168.203.107@tcp:250:kworker.0 00000100:00080000:0.0:1789763692.919271:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763692.919287:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763692.919289:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763692.919296:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 3126 00000100:00080000:0.0:1789763692.919301:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763692.919307:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789763692.919314:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763692.919333:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d1f80 x1876700024768128/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789763692.919369:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763692.919426:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763692/real 1789763692] req@ffffa06c091d0700 x1876700024767872/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763717 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763692.919449:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0700 x1876700024767872/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763717 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763692.919464:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763692.919469:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763692.919474:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024767872:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789763692.919502:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024768000:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763692.919527:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763692.919541:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763692/real 1789763692] req@ffffa06c091d0a80 x1876700024768000/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763717 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763692.919557:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0a80 x1876700024768000/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763717 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763692.919588:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763692.919593:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763692.919597:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024768000:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789763692.919605:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024768128:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763692.919630:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763692.919643:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763692/real 1789763692] req@ffffa06c091d1f80 x1876700024768128/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763717 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763692.919657:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d1f80 x1876700024768128/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763717 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763692.919666:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763692.919669:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763692.919673:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024768128:192.168.203.107@tcp:38:kworker.0 00010000:00010000:1.0:1789763693.175057:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763693.430973:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763694.199066:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763694.454956:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763695.222998:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763695.478989:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789763696.247104:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763696.502956:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789763697.271034:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763697.526957:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:3.0:1789763698.039029:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763698.039041:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763698.039046:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763698.039061:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 3131 00000100:00080000:3.0:1789763698.039102:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763698.039139:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:3.0:1789763698.039196:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872fb80 x1876700024768256/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763698.039247:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763698.039252:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763698.039255:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763698.039261:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 3131 00000100:00080000:3.0:1789763698.039268:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763698.039275:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789763698.039289:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763698.039303:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024768256:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789763698.039315:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872ed80 x1876700024768384/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763698.039335:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763698.039340:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763698.039343:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763698.039349:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 3131 00000100:00080000:3.0:1789763698.039354:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763698.039359:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789763698.039365:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763698.039378:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872f480 x1876700024768512/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789763698.039393:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789763698.039445:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763698/real 1789763698] req@ffffa06c1872fb80 x1876700024768256/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763723 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763698.039469:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872fb80 x1876700024768256/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763723 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763698.039484:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763698.039489:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789763698.039494:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024768256:192.168.203.107@tcp:250:kworker.0 00000100:00100000:0.0:1789763698.039537:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024768384:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789763698.039582:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789763698.039598:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763698/real 1789763698] req@ffffa06c1872ed80 x1876700024768384/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763723 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763698.039620:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872ed80 x1876700024768384/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763723 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763698.039637:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763698.039644:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789763698.039651:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024768384:192.168.203.107@tcp:38:kworker.0 00000100:00100000:0.0:1789763698.039679:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024768512:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789763698.039722:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789763698.039741:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763698/real 1789763698] req@ffffa06c1872f480 x1876700024768512/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763723 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763698.039757:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872f480 x1876700024768512/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763723 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763698.039767:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763698.039771:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789763698.039775:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024768512:192.168.203.107@tcp:38:kworker.0 00010000:00010000:2.0:1789763698.295058:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763698.550952:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763699.319033:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763699.574988:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789763700.343080:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763700.598996:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789763701.367025:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763701.622901:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763702.391045:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763702.646916:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:3.0:1789763703.158939:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763703.158953:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763703.158958:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763703.158970:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 3136 00000100:00080000:3.0:1789763703.158984:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763703.158993:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:3.0:1789763703.159038:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872ea00 x1876700024768640/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763703.159115:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763703.159123:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763703.159126:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763703.159134:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 3136 00000100:00080000:3.0:1789763703.159141:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763703.159151:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789763703.159167:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763703.159193:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024768640:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789763703.159197:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872dc00 x1876700024768768/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763703.159224:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763703.159228:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763703.159245:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763703.159252:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 3136 00000100:00080000:3.0:1789763703.159258:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763703.159265:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789763703.159272:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763703.159290:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872e680 x1876700024768896/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789763703.159301:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789763703.159357:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763703/real 1789763703] req@ffffa06c1872ea00 x1876700024768640/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763728 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763703.159414:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872ea00 x1876700024768640/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763728 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763703.159434:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763703.159441:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789763703.159449:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024768640:192.168.203.107@tcp:250:kworker.0 00000100:00100000:0.0:1789763703.159484:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024768768:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789763703.159517:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789763703.159548:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763703/real 1789763703] req@ffffa06c1872dc00 x1876700024768768/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763728 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763703.159564:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872dc00 x1876700024768768/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763728 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763703.159575:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763703.159579:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789763703.159583:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024768768:192.168.203.107@tcp:38:kworker.0 00000100:00100000:0.0:1789763703.159590:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024768896:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789763703.159611:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789763703.159622:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763703/real 1789763703] req@ffffa06c1872e680 x1876700024768896/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763728 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763703.159636:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872e680 x1876700024768896/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763728 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763703.159672:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763703.159676:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789763703.159680:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024768896:192.168.203.107@tcp:38:kworker.0 00010000:00010000:1.0:1789763703.414959:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763703.670913:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789763704.439116:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763704.694947:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763705.463032:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763705.718975:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789763706.487147:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763706.742974:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763707.511017:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763707.766963:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:1.0:1789763708.279078:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763708.279091:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789763708.279096:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763708.279109:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 3142 00000100:00080000:1.0:1789763708.279123:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763708.279133:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789763708.279182:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2c700 x1876700024769024/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763708.279225:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763708.279234:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763708.279237:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763708.279245:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 3142 00000100:00080000:1.0:1789763708.279252:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763708.279259:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789763708.279277:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763708.279302:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2ca80 x1876700024769152/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789763708.279320:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024769024:192.168.203.107@tcp:250:kworker.0 00000100:00080000:1.0:1789763708.279323:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763708.279329:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763708.279355:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763708.279362:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 3142 00000100:00080000:1.0:1789763708.279370:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763708.279377:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789763708.279387:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763708.279406:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2ed80 x1876700024769280/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789763708.279410:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789763708.279470:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763708/real 1789763708] req@ffffa06c08e2c700 x1876700024769024/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763733 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763708.279504:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2c700 x1876700024769024/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763733 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763708.279522:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763708.279527:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789763708.279532:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024769024:192.168.203.107@tcp:250:kworker.0 00000100:00100000:0.0:1789763708.279563:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024769152:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789763708.279597:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789763708.279613:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763708/real 1789763708] req@ffffa06c08e2ca80 x1876700024769152/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763733 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763708.279634:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2ca80 x1876700024769152/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763733 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763708.279646:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763708.279650:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789763708.279654:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024769152:192.168.203.107@tcp:38:kworker.0 00000100:00100000:0.0:1789763708.279675:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024769280:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789763708.279702:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789763708.279716:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763708/real 1789763708] req@ffffa06c08e2ed80 x1876700024769280/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763733 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763708.279731:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2ed80 x1876700024769280/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763733 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763708.279741:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763708.279744:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789763708.279748:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024769280:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789763708.535064:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763708.790986:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763709.559100:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763709.815041:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763710.583132:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763710.838975:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763711.607061:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763711.862963:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789763712.631079:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763712.886983:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789763713.399021:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763713.399033:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763713.399038:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763713.399051:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 3147 00000100:00080000:0.0:1789763713.399064:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763713.399073:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789763713.399132:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d3100 x1876700024769408/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763713.399171:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763713.399176:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763713.399179:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763713.399184:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 3147 00000100:00080000:0.0:1789763713.399188:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763713.399216:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789763713.399230:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763713.399259:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0380 x1876700024769536/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763713.399278:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1789763713.399282:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024769408:192.168.203.107@tcp:250:kworker.0 00000100:00080000:0.0:1789763713.399285:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763713.399292:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763713.399308:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 3147 00000100:00080000:0.0:1789763713.399313:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763713.399318:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789763713.399324:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763713.399340:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d2680 x1876700024769664/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789763713.399404:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763713.399472:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763713/real 1789763713] req@ffffa06c091d3100 x1876700024769408/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763738 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763713.399500:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d3100 x1876700024769408/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763738 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763713.399527:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763713.399533:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763713.399538:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024769408:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789763713.399577:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024769536:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763713.399621:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763713.399655:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763713/real 1789763713] req@ffffa06c091d0380 x1876700024769536/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763738 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763713.399678:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0380 x1876700024769536/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763738 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763713.399692:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763713.399698:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763713.399703:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024769536:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789763713.399715:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024769664:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763713.399842:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763713.399902:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763713/real 1789763713] req@ffffa06c091d2680 x1876700024769664/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763738 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763713.399919:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d2680 x1876700024769664/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763738 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763713.399930:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763713.399935:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763713.399939:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024769664:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789763713.655093:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763713.910971:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763714.679057:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763714.934944:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789763715.703129:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763715.959017:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763716.727049:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763716.983063:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763717.751125:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763718.007058:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:3.0:1789763718.519062:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763718.519077:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763718.519084:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763718.519101:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 3152 00000100:00080000:3.0:1789763718.519118:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763718.519129:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:3.0:1789763718.519196:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872c700 x1876700024769792/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763718.519256:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763718.519265:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763718.519269:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763718.519277:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 3152 00000100:00080000:3.0:1789763718.519285:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763718.519293:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789763718.519312:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763718.519318:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024769792:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789763718.519350:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872f100 x1876700024769920/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763718.519418:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763718.519424:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763718.519431:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1789763718.519432:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00080000:3.0:1789763718.519447:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 3152 00000100:00080000:3.0:1789763718.519461:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763718.519472:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789763718.519485:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763718.519509:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763718/real 1789763718] req@ffffa06c1872c700 x1876700024769792/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763743 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763718.519521:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872ca80 x1876700024770048/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789763718.519564:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872c700 x1876700024769792/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763743 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763718.519593:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763718.519604:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789763718.519614:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024769792:192.168.203.107@tcp:250:kworker.0 00000100:00100000:0.0:1789763718.519686:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024769920:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789763718.519738:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789763718.519768:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763718/real 1789763718] req@ffffa06c1872f100 x1876700024769920/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763743 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763718.519797:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872f100 x1876700024769920/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763743 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763718.519815:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763718.519823:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789763718.519831:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024769920:192.168.203.107@tcp:38:kworker.0 00000100:00100000:0.0:1789763718.519882:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024770048:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789763718.519929:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789763718.519949:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763718/real 1789763718] req@ffffa06c1872ca80 x1876700024770048/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763743 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763718.520005:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872ca80 x1876700024770048/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763743 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763718.520022:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763718.520029:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789763718.520036:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024770048:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789763718.775103:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763719.031018:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763719.799046:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763720.055049:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763720.822990:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763721.078898:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789763721.847072:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763722.102955:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763722.871060:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763723.127019:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789763723.638995:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763723.639009:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763723.639014:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763723.639027:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 3157 00000100:00080000:2.0:1789763723.639042:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763723.639051:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789763723.639103:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50700 x1876700024770176/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763723.639144:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763723.639149:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763723.639152:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763723.639157:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 3157 00000100:00080000:2.0:1789763723.639162:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763723.639166:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789763723.639200:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763723.639220:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50a80 x1876700024770304/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789763723.639238:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024770176:192.168.203.107@tcp:250:kworker.0 00000100:00080000:2.0:1789763723.639242:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763723.639255:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763723.639258:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763723.639264:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 3157 00000100:00080000:2.0:1789763723.639271:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763723.639276:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789763723.639283:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763723.639301:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e53480 x1876700024770432/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789763723.639336:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789763723.639393:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763723/real 1789763723] req@ffffa06c07e50700 x1876700024770176/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763748 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763723.639444:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50700 x1876700024770176/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763748 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763723.639466:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763723.639475:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789763723.639486:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024770176:192.168.203.107@tcp:250:kworker.0 00000100:00100000:0.0:1789763723.639525:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024770304:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789763723.639565:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789763723.639588:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763723/real 1789763723] req@ffffa06c07e50a80 x1876700024770304/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763748 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763723.639605:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50a80 x1876700024770304/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763748 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763723.639616:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763723.639621:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789763723.639625:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024770304:192.168.203.107@tcp:38:kworker.0 00000100:00100000:0.0:1789763723.639634:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e53480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024770432:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789763723.639663:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789763723.639684:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763723/real 1789763723] req@ffffa06c07e53480 x1876700024770432/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763748 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763723.639710:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e53480 x1876700024770432/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763748 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763723.639753:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763723.639759:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789763723.639764:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e53480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024770432:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789763723.895091:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763724.151006:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789763724.919090:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763725.175003:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763725.943084:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763726.199002:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763726.967019:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763727.223013:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789763727.991128:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763728.246982:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:3.0:1789763728.759078:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763728.759137:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763728.759144:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763728.759161:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 3162 00000100:00080000:3.0:1789763728.759179:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763728.759191:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:3.0:1789763728.759251:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872d180 x1876700024770560/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763728.759298:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763728.759303:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763728.759306:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763728.759311:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 3162 00000100:00080000:3.0:1789763728.759316:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763728.759321:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789763728.759335:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763728.759351:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872e300 x1876700024770688/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763728.759366:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763728.759371:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763728.759373:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763728.759377:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 3162 00000100:00080000:3.0:1789763728.759380:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763728.759385:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789763728.759392:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763728.759404:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872ce00 x1876700024770816/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789763728.759453:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024770560:192.168.203.107@tcp:250:kworker.0 00000800:00000100:0.0:1789763728.759550:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789763728.759605:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763728/real 1789763728] req@ffffa06c1872d180 x1876700024770560/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763753 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763728.759630:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872d180 x1876700024770560/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763753 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763728.759646:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763728.759651:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789763728.759657:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024770560:192.168.203.107@tcp:250:kworker.0 00000100:00100000:0.0:1789763728.759666:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024770688:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789763728.759692:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789763728.759706:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763728/real 1789763728] req@ffffa06c1872e300 x1876700024770688/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763753 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763728.759721:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872e300 x1876700024770688/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763753 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763728.759731:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763728.759735:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789763728.759755:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024770688:192.168.203.107@tcp:38:kworker.0 00000100:00100000:0.0:1789763728.759788:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024770816:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789763728.759826:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789763728.759938:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763728/real 1789763728] req@ffffa06c1872ce00 x1876700024770816/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763753 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763728.759966:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872ce00 x1876700024770816/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763753 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763728.759979:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763728.759983:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789763728.759988:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024770816:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789763729.015112:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763729.271049:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763730.039087:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763730.294969:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789763731.063091:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763731.318971:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789763732.087051:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763732.342977:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763733.111053:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763733.367012:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:1.0:1789763733.879021:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763733.879033:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789763733.879039:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763733.879052:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 3167 00000100:00080000:1.0:1789763733.879066:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763733.879075:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789763733.879120:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2d180 x1876700024770944/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763733.879155:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763733.879160:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763733.879163:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763733.879168:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 3167 00000100:00080000:1.0:1789763733.879173:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763733.879177:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789763733.879188:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763733.879216:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2ed80 x1876700024771072/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763733.879253:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1789763733.879256:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024770944:192.168.203.107@tcp:250:kworker.0 00000100:00080000:1.0:1789763733.879264:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763733.879273:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763733.879282:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 3167 00000100:00080000:1.0:1789763733.879289:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763733.879295:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789763733.879304:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763733.879325:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2ca80 x1876700024771200/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789763733.879362:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789763733.879434:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763733/real 1789763733] req@ffffa06c08e2d180 x1876700024770944/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763758 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763733.879468:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2d180 x1876700024770944/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763758 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763733.879482:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763733.879487:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789763733.879493:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024770944:192.168.203.107@tcp:250:kworker.0 00000100:00100000:0.0:1789763733.879524:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024771072:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789763733.879574:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789763733.879591:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763733/real 1789763733] req@ffffa06c08e2ed80 x1876700024771072/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763758 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763733.879607:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2ed80 x1876700024771072/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763758 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763733.879618:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763733.879622:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789763733.879627:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024771072:192.168.203.107@tcp:38:kworker.0 00000100:00100000:0.0:1789763733.879634:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024771200:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789763733.879653:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789763733.879664:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763733/real 1789763733] req@ffffa06c08e2ca80 x1876700024771200/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763758 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763733.879678:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2ca80 x1876700024771200/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763758 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763733.879687:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763733.879691:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789763733.879694:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024771200:192.168.203.107@tcp:38:kworker.0 00010000:00010000:2.0:1789763734.135049:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763734.390965:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763735.159037:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763735.414988:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763736.183067:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763736.439001:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789763737.207195:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763737.462963:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763738.231156:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763738.486980:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:3.0:1789763738.999044:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763738.999057:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763738.999062:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763738.999074:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 3172 00000100:00080000:3.0:1789763738.999088:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763738.999097:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:3.0:1789763738.999168:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872c380 x1876700024771328/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763738.999223:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763738.999231:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763738.999235:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763738.999269:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 3172 00000100:00080000:3.0:1789763738.999294:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763738.999301:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789763738.999314:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763738.999336:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872d880 x1876700024771456/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763738.999356:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1789763738.999360:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024771328:192.168.203.107@tcp:250:kworker.0 00000100:00080000:3.0:1789763738.999362:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763738.999369:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763738.999398:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 3172 00000100:00080000:3.0:1789763738.999402:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763738.999408:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789763738.999415:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763738.999435:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872df80 x1876700024771584/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789763738.999459:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789763738.999515:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763738/real 1789763738] req@ffffa06c1872c380 x1876700024771328/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763763 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763738.999549:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872c380 x1876700024771328/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763763 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763738.999559:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763738.999563:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789763738.999567:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024771328:192.168.203.107@tcp:250:kworker.0 00000100:00100000:0.0:1789763738.999591:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024771456:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789763738.999622:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789763738.999640:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763738/real 1789763738] req@ffffa06c1872d880 x1876700024771456/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763763 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763738.999659:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872d880 x1876700024771456/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763763 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763738.999696:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763738.999703:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789763738.999710:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024771456:192.168.203.107@tcp:38:kworker.0 00000100:00100000:0.0:1789763738.999723:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024771584:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789763738.999763:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789763738.999787:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763738/real 1789763738] req@ffffa06c1872df80 x1876700024771584/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763763 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763738.999838:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872df80 x1876700024771584/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763763 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763738.999930:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763738.999937:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789763738.999942:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024771584:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789763739.255064:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763739.510972:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789763740.279185:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763740.534995:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789763741.303038:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763741.558968:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763742.327022:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763742.582955:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789763743.351107:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763743.606951:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789763744.119062:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763744.119078:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763744.119086:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763744.119102:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 3177 00000100:00080000:0.0:1789763744.119118:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763744.119130:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789763744.119188:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d2a00 x1876700024771712/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763744.119234:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763744.119242:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763744.119245:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763744.119254:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 3177 00000100:00080000:0.0:1789763744.119262:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763744.119269:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789763744.119282:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763744.119292:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024771712:192.168.203.107@tcp:250:kworker.0 00000100:00100000:0.0:1789763744.119309:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0e00 x1876700024771840/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763744.119328:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763744.119333:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763744.119354:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763744.119360:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 3177 00000100:00080000:0.0:1789763744.119367:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763744.119372:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000800:00000100:3.0:1789763744.119380:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000008:00100000:0.0:1789763744.119382:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763744.119405:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d1500 x1876700024771968/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763744.119441:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763744/real 1789763744] req@ffffa06c091d2a00 x1876700024771712/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763769 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763744.119479:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d2a00 x1876700024771712/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763769 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763744.119494:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763744.119502:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763744.119509:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024771712:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789763744.119550:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024771840:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763744.119595:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763744.119615:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763744/real 1789763744] req@ffffa06c091d0e00 x1876700024771840/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763769 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763744.119631:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0e00 x1876700024771840/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763769 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763744.119641:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763744.119644:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763744.119648:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024771840:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789763744.119655:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024771968:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763744.119679:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763744.119690:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763744/real 1789763744] req@ffffa06c091d1500 x1876700024771968/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763769 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763744.119703:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d1500 x1876700024771968/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763769 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763744.119729:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763744.119733:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763744.119737:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024771968:192.168.203.107@tcp:38:kworker.0 00010000:00010000:1.0:1789763744.375043:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763744.630970:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763745.399059:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763745.654970:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789763746.423075:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763746.678976:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789763747.447103:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763747.702974:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763748.471064:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763748.726962:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789763749.239021:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763749.239034:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763749.239040:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763749.239055:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 3183 00000100:00080000:0.0:1789763749.239070:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763749.239081:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789763749.239131:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d2300 x1876700024772096/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763749.239166:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763749.239171:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763749.239174:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763749.239179:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 3183 00000100:00080000:0.0:1789763749.239184:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763749.239189:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789763749.239201:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763749.239218:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0000 x1876700024772224/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763749.239254:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024772096:192.168.203.107@tcp:250:kworker.0 00000100:00080000:0.0:1789763749.239304:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763749.239312:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763749.239316:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763749.239328:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 3183 00000100:00080000:0.0:1789763749.239338:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763749.239349:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789763749.239362:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763749.239394:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d3b80 x1876700024772352/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763749.239484:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024772224:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789763749.239538:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024772352:192.168.203.107@tcp:38:kworker.0 00010000:00010000:0.0:1789763749.495108:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763749.750953:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763750.519112:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763750.774978:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763751.543048:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763751.799031:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000400:00000100:2.0:1789763752.311170:0:2330:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.107:988 00000400:00000100:2.0:1789763752.311211:0:2330:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.107@tcp at host 192.168.203.107 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1789763752.311227:0:2330:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.107@tcp 00000800:00000100:2.0:1789763752.311282:0:2330:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.107@tcp 00000800:00000100:2.0:1789763752.311291:0:2330:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.107@tcp 00000100:00100000:3.0:1789763752.311330:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763749/real 1789763752] req@ffffa06c091d0000 x1876700024772224/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763774 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763752.311377:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0000 x1876700024772224/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763774 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763752.311394:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763752.311412:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763752.311418:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024772224:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789763752.311430:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763749/real 1789763752] req@ffffa06c091d3b80 x1876700024772352/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763774 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763752.311448:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d3b80 x1876700024772352/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763774 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763752.311474:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763752.311477:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763752.311480:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024772352:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789763752.311486:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763749/real 1789763752] req@ffffa06c091d2300 x1876700024772096/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763774 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763752.311498:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d2300 x1876700024772096/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763774 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763752.311507:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763752.311511:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763752.311515:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024772096:192.168.203.107@tcp:250:kworker.0 00010000:00010000:0.0:1789763752.567090:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763752.822949:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763753.591045:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763753.846987:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789763754.359017:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763754.359041:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763754.359046:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763754.359057:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 3188 00000100:00080000:2.0:1789763754.359071:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763754.359080:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789763754.359178:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51c00 x1876700024772480/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763754.359230:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763754.359237:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763754.359240:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763754.359248:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 3188 00000100:00080000:2.0:1789763754.359273:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763754.359280:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789763754.359294:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763754.359304:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024772480:192.168.203.107@tcp:250:kworker.0 00000100:00100000:2.0:1789763754.359321:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e53800 x1876700024772608/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763754.359343:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763754.359349:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763754.359352:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763754.359358:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 3188 00000100:00080000:2.0:1789763754.359364:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763754.359371:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789763754.359380:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763754.359400:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e52680 x1876700024772736/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789763754.359410:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763754.359488:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763754/real 1789763754] req@ffffa06c07e51c00 x1876700024772480/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763779 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763754.359531:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51c00 x1876700024772480/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763779 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763754.359548:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763754.359555:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763754.359561:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024772480:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789763754.359592:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e53800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024772608:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763754.359656:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763754.359679:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763754/real 1789763754] req@ffffa06c07e53800 x1876700024772608/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763779 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763754.359716:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e53800 x1876700024772608/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763779 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763754.359731:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763754.359752:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763754.359759:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e53800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024772608:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789763754.359769:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e52680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024772736:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763754.359807:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763754.359828:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763754/real 1789763754] req@ffffa06c07e52680 x1876700024772736/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763779 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763754.359885:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e52680 x1876700024772736/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763779 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763754.359920:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763754.359926:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763754.359931:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e52680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024772736:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789763754.615033:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763754.870990:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789763755.639143:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763755.894963:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763756.663115:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763756.919017:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763757.687018:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763757.942937:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789763758.711091:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763758.966929:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789763759.478988:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763759.478999:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763759.479004:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763759.479015:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 3193 00000100:00080000:0.0:1789763759.479028:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763759.479035:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789763759.479081:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d2300 x1876700024772864/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763759.479113:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763759.479117:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763759.479119:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763759.479123:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 3193 00000100:00080000:0.0:1789763759.479161:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763759.479169:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:3.0:1789763759.479183:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024772864:192.168.203.107@tcp:250:kworker.0 00000008:00100000:0.0:1789763759.479184:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763759.479217:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d3b80 x1876700024772992/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763759.479234:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763759.479238:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763759.479241:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763759.479245:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 3193 00000100:00080000:0.0:1789763759.479268:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763759.479274:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789763759.479282:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1789763759.479301:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789763759.479302:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0000 x1876700024773120/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763759.479378:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763759/real 1789763759] req@ffffa06c091d2300 x1876700024772864/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763784 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763759.479405:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d2300 x1876700024772864/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763784 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763759.479434:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763759.479439:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763759.479443:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024772864:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789763759.479469:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024772992:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763759.479515:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763759.479531:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763759/real 1789763759] req@ffffa06c091d3b80 x1876700024772992/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763784 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763759.479561:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d3b80 x1876700024772992/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763784 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763759.479573:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763759.479577:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763759.479581:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024772992:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789763759.479588:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024773120:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763759.479613:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763759.479625:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763759/real 1789763759] req@ffffa06c091d0000 x1876700024773120/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763784 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763759.479639:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0000 x1876700024773120/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763784 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763759.479649:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763759.479699:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763759.479704:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024773120:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789763759.734997:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763759.990911:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763760.759023:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763761.014974:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789763761.783031:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763762.038934:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763762.806964:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763763.062916:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763763.831030:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763764.086990:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789763764.599016:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763764.599029:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763764.599035:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763764.599048:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 3198 00000100:00080000:2.0:1789763764.599070:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763764.599079:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789763764.599153:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50380 x1876700024773248/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763764.599200:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763764.599207:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763764.599209:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763764.599216:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 3198 00000100:00080000:2.0:1789763764.599222:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763764.599229:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789763764.599242:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763764.599252:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024773248:192.168.203.107@tcp:250:kworker.0 00000100:00100000:2.0:1789763764.599265:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e53b80 x1876700024773376/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763764.599284:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763764.599288:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763764.599290:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763764.599314:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 3198 00000100:00080000:2.0:1789763764.599320:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763764.599326:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789763764.599333:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1789763764.599344:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789763764.599351:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51f80 x1876700024773504/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763764.599425:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763764/real 1789763764] req@ffffa06c07e50380 x1876700024773248/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763789 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763764.599461:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50380 x1876700024773248/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763789 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763764.599478:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763764.599483:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763764.599488:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024773248:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789763764.599518:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e53b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024773376:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763764.599550:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763764.599566:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763764/real 1789763764] req@ffffa06c07e53b80 x1876700024773376/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763789 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763764.599580:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e53b80 x1876700024773376/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763789 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763764.599590:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763764.599594:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763764.599598:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e53b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024773376:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789763764.599605:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024773504:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763764.599658:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763764.599673:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763764/real 1789763764] req@ffffa06c07e51f80 x1876700024773504/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763789 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763764.599712:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51f80 x1876700024773504/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763789 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763764.599727:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763764.599733:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763764.599739:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024773504:192.168.203.107@tcp:38:kworker.0 00010000:00010000:2.0:1789763764.855036:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789763765.111037:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763765.879085:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789763766.134907:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763766.903012:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789763767.158932:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763767.926984:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789763768.182966:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763768.951028:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789763769.206917:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:1.0:1789763769.718987:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763769.718998:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789763769.719002:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763769.719013:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 3203 00000100:00080000:1.0:1789763769.719025:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763769.719032:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789763769.719076:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2c700 x1876700024773632/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763769.719101:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763769.719104:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763769.719106:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763769.719124:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 3203 00000100:00080000:1.0:1789763769.719127:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763769.719130:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:3.0:1789763769.719140:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024773632:192.168.203.107@tcp:250:kworker.0 00000008:00100000:1.0:1789763769.719141:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763769.719157:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2c000 x1876700024773760/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763769.719176:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763769.719179:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763769.719181:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763769.719184:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 3203 00000100:00080000:1.0:1789763769.719187:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763769.719189:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789763769.719193:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763769.719202:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2e300 x1876700024773888/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789763769.719217:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763769.719270:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763769/real 1789763769] req@ffffa06c08e2c700 x1876700024773632/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763794 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763769.719289:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2c700 x1876700024773632/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763794 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763769.719298:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763769.719301:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763769.719304:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024773632:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789763769.719328:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024773760:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763769.719346:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763769.719356:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763769/real 1789763769] req@ffffa06c08e2c000 x1876700024773760/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763794 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763769.719376:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2c000 x1876700024773760/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763794 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763769.719382:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763769.719385:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763769.719387:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024773760:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789763769.719391:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024773888:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763769.719405:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763769.719412:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763769/real 1789763769] req@ffffa06c08e2e300 x1876700024773888/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763794 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763769.719420:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2e300 x1876700024773888/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763794 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763769.719425:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763769.719427:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763769.719429:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024773888:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789763769.975031:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789763770.230979:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763770.999019:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763771.255049:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763772.023086:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763772.278997:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763773.046978:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789763773.303011:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763774.071041:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789763774.327042:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789763774.839081:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763774.839094:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763774.839099:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763774.839112:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 3208 00000100:00080000:0.0:1789763774.839153:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763774.839167:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789763774.839224:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d1500 x1876700024774016/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763774.839287:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763774.839295:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763774.839298:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763774.839306:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 3208 00000100:00080000:0.0:1789763774.839313:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763774.839321:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789763774.839335:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763774.839357:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0e00 x1876700024774144/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763774.839378:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024774016:192.168.203.107@tcp:250:kworker.0 00000100:00080000:0.0:1789763774.839383:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763774.839393:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763774.839397:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763774.839403:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 3208 00000100:00080000:0.0:1789763774.839407:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763774.839411:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789763774.839423:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763774.839439:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d2a00 x1876700024774272/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789763774.839526:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763774.839629:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763774/real 1789763774] req@ffffa06c091d1500 x1876700024774016/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763799 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763774.839671:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d1500 x1876700024774016/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763799 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763774.839686:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763774.839692:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763774.839697:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024774016:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789763774.839733:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024774144:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763774.839762:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763774.839794:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763774/real 1789763774] req@ffffa06c091d0e00 x1876700024774144/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763799 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763774.839806:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0e00 x1876700024774144/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763799 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763774.839815:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763774.839819:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763774.839822:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024774144:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789763774.839828:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024774272:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763774.839846:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763774.839855:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763774/real 1789763774] req@ffffa06c091d2a00 x1876700024774272/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763799 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763774.839881:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d2a00 x1876700024774272/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763799 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763774.839938:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763774.839945:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763774.839950:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024774272:192.168.203.107@tcp:38:kworker.0 00010000:00010000:2.0:1789763775.094989:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789763775.351082:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763776.119029:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789763776.375049:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763777.142978:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789763777.398935:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763778.167028:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789763778.422956:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763779.191013:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789763779.446952:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789763779.958989:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763779.959000:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763779.959004:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763779.959015:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 3213 00000100:00080000:2.0:1789763779.959027:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763779.959035:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789763779.959082:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50e00 x1876700024774400/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763779.959114:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763779.959118:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763779.959119:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763779.959123:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 3213 00000100:00080000:2.0:1789763779.959125:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763779.959127:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789763779.959135:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763779.959145:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51500 x1876700024774528/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763779.959173:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763779.959176:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763779.959177:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763779.959180:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 3213 00000100:00080000:2.0:1789763779.959183:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763779.959189:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:3.0:1789763779.959190:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024774400:192.168.203.107@tcp:250:kworker.0 00000008:00100000:2.0:1789763779.959193:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763779.959203:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50000 x1876700024774656/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789763779.959265:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763779.959312:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763779/real 1789763779] req@ffffa06c07e50e00 x1876700024774400/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763804 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763779.959326:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50e00 x1876700024774400/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763804 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763779.959335:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763779.959338:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763779.959341:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024774400:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789763779.959346:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024774528:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763779.959361:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763779.959369:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763779/real 1789763779] req@ffffa06c07e51500 x1876700024774528/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763804 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763779.959390:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51500 x1876700024774528/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763804 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763779.959397:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763779.959399:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763779.959401:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024774528:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789763779.959425:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024774656:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763779.959438:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763779.959445:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763779/real 1789763779] req@ffffa06c07e50000 x1876700024774656/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763804 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763779.959461:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50000 x1876700024774656/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763804 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763779.959467:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763779.959470:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763779.959472:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024774656:192.168.203.107@tcp:38:kworker.0 00010000:00010000:1.0:1789763780.215042:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789763780.470950:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789763781.238980:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763781.494998:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789763782.263116:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763782.518957:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789763783.286985:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763783.542945:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789763784.310986:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763784.566913:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789763785.078979:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763785.078993:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763785.078998:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763785.079012:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 3218 00000100:00080000:2.0:1789763785.079025:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763785.079034:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789763785.079089:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51180 x1876700024774784/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763785.079120:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763785.079123:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763785.079125:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763785.079129:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 3218 00000100:00080000:2.0:1789763785.079132:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763785.079154:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789763785.079164:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763785.079174:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e52300 x1876700024774912/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763785.079184:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763785.079187:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763785.079190:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763785.079192:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 3218 00000100:00080000:2.0:1789763785.079195:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763785.079198:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789763785.079201:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763785.079209:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024774784:192.168.203.107@tcp:250:kworker.0 00000100:00100000:2.0:1789763785.079215:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51880 x1876700024775040/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789763785.079287:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763785.079354:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763785/real 1789763785] req@ffffa06c07e51180 x1876700024774784/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763810 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763785.079425:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51180 x1876700024774784/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763810 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763785.079434:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763785.079438:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763785.079443:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024774784:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789763785.079463:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e52300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024774912:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763785.079506:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763785.079518:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763785/real 1789763785] req@ffffa06c07e52300 x1876700024774912/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763810 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763785.079527:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e52300 x1876700024774912/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763810 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763785.079533:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763785.079536:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763785.079538:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e52300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024774912:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789763785.079558:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024775040:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763785.079571:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763785.079579:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763785/real 1789763785] req@ffffa06c07e51880 x1876700024775040/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763810 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763785.079587:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51880 x1876700024775040/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763810 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763785.079601:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763785.079605:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763785.079609:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024775040:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789763785.334955:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763785.590945:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789763786.359016:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763786.614914:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789763787.382983:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763787.638962:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763788.406986:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763788.662915:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789763789.431091:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763789.686925:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789763790.198968:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763790.198979:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763790.198983:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763790.198995:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 3224 00000100:00080000:2.0:1789763790.199006:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763790.199031:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789763790.199094:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51880 x1876700024775168/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763790.199119:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763790.199122:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763790.199141:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763790.199146:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 3224 00000100:00080000:2.0:1789763790.199150:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763790.199154:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789763790.199165:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763790.199172:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024775168:192.168.203.107@tcp:250:kworker.0 00000100:00100000:2.0:1789763790.199180:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e52300 x1876700024775296/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763790.199191:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763790.199193:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763790.199195:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763790.199198:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 3224 00000100:00080000:2.0:1789763790.199200:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763790.199203:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789763790.199206:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763790.199215:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51180 x1876700024775424/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789763790.199250:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763790.199296:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763790/real 1789763790] req@ffffa06c07e51880 x1876700024775168/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763815 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763790.199312:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51880 x1876700024775168/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763815 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763790.199321:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763790.199325:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763790.199328:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024775168:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789763790.199351:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e52300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024775296:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763790.199374:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763790.199389:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763790/real 1789763790] req@ffffa06c07e52300 x1876700024775296/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763815 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763790.199419:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e52300 x1876700024775296/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763815 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763790.199437:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763790.199440:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763790.199443:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e52300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024775296:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789763790.199448:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024775424:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763790.199466:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763790.199475:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763790/real 1789763790] req@ffffa06c07e51180 x1876700024775424/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763815 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763790.199483:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51180 x1876700024775424/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763815 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763790.199489:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763790.199491:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763790.199493:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024775424:192.168.203.107@tcp:38:kworker.0 00010000:00010000:1.0:1789763790.454994:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763790.710948:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763791.479003:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763791.734968:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789763792.502972:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763792.758956:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763793.526993:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763793.782939:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763794.551003:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763794.806956:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789763795.318920:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763795.318930:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763795.318933:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763795.318942:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 3229 00000100:00080000:2.0:1789763795.318951:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763795.318957:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789763795.318991:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50000 x1876700024775552/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763795.319029:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763795.319032:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763795.319034:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763795.319037:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 3229 00000100:00080000:2.0:1789763795.319040:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763795.319043:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789763795.319050:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763795.319061:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51500 x1876700024775680/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763795.319069:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763795.319072:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763795.319073:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763795.319078:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 3229 00000100:00100000:3.0:1789763795.319081:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024775552:192.168.203.107@tcp:250:kworker.0 00000100:00080000:2.0:1789763795.319082:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763795.319086:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789763795.319089:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763795.319101:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50e00 x1876700024775808/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789763795.319155:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763795.319195:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763795/real 1789763795] req@ffffa06c07e50000 x1876700024775552/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763820 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763795.319209:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50000 x1876700024775552/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763820 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763795.319217:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763795.319220:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763795.319223:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024775552:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789763795.319228:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024775680:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763795.319242:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763795.319250:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763795/real 1789763795] req@ffffa06c07e51500 x1876700024775680/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763820 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763795.319257:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51500 x1876700024775680/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763820 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763795.319263:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763795.319265:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763795.319267:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024775680:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789763795.319284:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024775808:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763795.319294:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763795.319300:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763795/real 1789763795] req@ffffa06c07e50e00 x1876700024775808/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763820 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763795.319317:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50e00 x1876700024775808/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763820 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763795.319323:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763795.319325:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763795.319327:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024775808:192.168.203.107@tcp:38:kworker.0 00010000:00010000:1.0:1789763795.574956:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763795.830950:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763796.598974:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763796.854911:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763797.622965:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763797.878924:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763798.647001:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763798.902907:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763799.670981:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763799.926934:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:1.0:1789763800.439025:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763800.439036:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789763800.439040:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763800.439051:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 3234 00000100:00080000:1.0:1789763800.439087:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763800.439094:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789763800.439134:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2d500 x1876700024775936/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763800.439162:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763800.439165:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763800.439167:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763800.439170:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 3234 00000100:00080000:1.0:1789763800.439172:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763800.439175:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789763800.439184:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763800.439195:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2e300 x1876700024776064/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763800.439205:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763800.439208:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763800.439211:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1789763800.439213:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024775936:192.168.203.107@tcp:250:kworker.0 00000100:00080000:1.0:1789763800.439216:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 3234 00000100:00080000:1.0:1789763800.439223:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763800.439226:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789763800.439229:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763800.439237:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2c000 x1876700024776192/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789763800.439291:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763800.439335:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763800/real 1789763800] req@ffffa06c08e2d500 x1876700024775936/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763825 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763800.439363:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2d500 x1876700024775936/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763825 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763800.439373:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763800.439377:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763800.439380:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024775936:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789763800.439417:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024776064:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763800.439436:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763800.439446:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763800/real 1789763800] req@ffffa06c08e2e300 x1876700024776064/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763825 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763800.439455:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2e300 x1876700024776064/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763825 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763800.439461:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763800.439464:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763800.439466:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024776064:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789763800.439470:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024776192:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763800.439484:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763800.439494:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763800/real 1789763800] req@ffffa06c08e2c000 x1876700024776192/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763825 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763800.439506:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2c000 x1876700024776192/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763825 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763800.439515:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763800.439519:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763800.439522:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024776192:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789763800.694996:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763800.950970:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789763801.718973:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763801.974931:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763802.742969:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763802.998930:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763803.766977:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763804.022940:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789763804.791001:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763805.046945:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789763805.558971:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763805.558981:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763805.558986:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763805.558997:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 3239 00000100:00080000:2.0:1789763805.559031:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763805.559039:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789763805.559080:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51f80 x1876700024776320/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763805.559108:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763805.559111:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763805.559113:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763805.559116:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 3239 00000100:00080000:2.0:1789763805.559118:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763805.559121:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789763805.559130:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763805.559139:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e53b80 x1876700024776448/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763805.559148:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763805.559151:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763805.559153:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763805.559156:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 3239 00000100:00080000:2.0:1789763805.559161:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763805.559164:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789763805.559168:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763805.559177:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50380 x1876700024776576/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763805.559198:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024776320:192.168.203.107@tcp:250:kworker.0 00000800:00000100:3.0:1789763805.559270:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763805.559306:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763805/real 1789763805] req@ffffa06c07e51f80 x1876700024776320/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763830 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763805.559338:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51f80 x1876700024776320/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763830 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763805.559351:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763805.559356:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763805.559360:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024776320:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789763805.559368:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e53b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024776448:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763805.559391:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763805.559402:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763805/real 1789763805] req@ffffa06c07e53b80 x1876700024776448/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763830 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763805.559411:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e53b80 x1876700024776448/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763830 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763805.559416:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763805.559419:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763805.559421:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e53b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024776448:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789763805.559441:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024776576:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763805.559478:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763805.559491:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763805/real 1789763805] req@ffffa06c07e50380 x1876700024776576/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763830 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763805.559503:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50380 x1876700024776576/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763830 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763805.559509:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763805.559511:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763805.559513:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024776576:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789763805.814989:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763806.070892:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763806.838961:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763807.094932:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763807.862965:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763808.118912:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763808.886999:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763809.142978:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763809.911049:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763810.166981:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789763810.678940:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763810.678975:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763810.678979:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763810.678990:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 3244 00000100:00080000:2.0:1789763810.679002:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763810.679010:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789763810.679050:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e52680 x1876700024776704/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763810.679078:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763810.679082:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763810.679083:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763810.679087:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 3244 00000100:00080000:2.0:1789763810.679089:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763810.679092:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789763810.679101:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763810.679111:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e53800 x1876700024776832/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763810.679120:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763810.679124:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763810.679125:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763810.679128:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 3244 00000100:00080000:2.0:1789763810.679129:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763810.679132:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789763810.679136:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763810.679143:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51c00 x1876700024776960/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763810.679176:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e52680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024776704:192.168.203.107@tcp:250:kworker.0 00000800:00000100:3.0:1789763810.679246:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763810.679302:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763810/real 1789763810] req@ffffa06c07e52680 x1876700024776704/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763835 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763810.679318:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e52680 x1876700024776704/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763835 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763810.679327:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763810.679345:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763810.679349:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e52680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024776704:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789763810.679358:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e53800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024776832:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763810.679385:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763810.679400:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763810/real 1789763810] req@ffffa06c07e53800 x1876700024776832/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763835 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763810.679413:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e53800 x1876700024776832/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763835 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763810.679423:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763810.679426:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763810.679429:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e53800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024776832:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789763810.679453:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024776960:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763810.679476:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763810.679490:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763810/real 1789763810] req@ffffa06c07e51c00 x1876700024776960/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763835 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763810.679502:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51c00 x1876700024776960/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763835 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763810.679511:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763810.679514:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763810.679517:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024776960:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789763810.934998:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763811.190980:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789763811.958998:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763812.214915:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789763812.983010:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763813.238977:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763814.006996:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763814.262960:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789763815.031019:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763815.286965:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:1.0:1789763815.798927:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763815.798934:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789763815.798938:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763815.798947:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 3249 00000100:00080000:1.0:1789763815.798955:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763815.798961:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789763815.798999:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2c700 x1876700024777088/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763815.799018:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763815.799021:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763815.799022:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763815.799025:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 3249 00000100:00080000:1.0:1789763815.799028:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763815.799031:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789763815.799039:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763815.799048:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2ca80 x1876700024777216/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763815.799058:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1789763815.799060:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024777088:192.168.203.107@tcp:250:kworker.0 00000100:00080000:1.0:1789763815.799061:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763815.799063:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763815.799070:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 3249 00000100:00080000:1.0:1789763815.799073:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763815.799075:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789763815.799097:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763815.799108:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2ed80 x1876700024777344/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763815.799159:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024777216:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789763815.799183:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024777344:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789763816.055070:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763816.311067:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789763817.078958:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763817.334953:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789763818.103033:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763818.358960:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000400:00000100:2.0:1789763818.871198:0:2331:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.107:988 00000400:00000100:2.0:1789763818.871279:0:2331:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.107@tcp at host 192.168.203.107 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1789763818.871296:0:2331:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.107@tcp 00000800:00000100:2.0:1789763818.871341:0:2331:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.107@tcp 00000800:00000100:2.0:1789763818.871348:0:2331:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.107@tcp 00000100:00100000:3.0:1789763818.871435:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763815/real 1789763818] req@ffffa06c08e2ca80 x1876700024777216/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763840 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763818.871496:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2ca80 x1876700024777216/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763840 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763818.871521:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763818.871531:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763818.871561:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024777216:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789763818.871580:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763815/real 1789763818] req@ffffa06c08e2ed80 x1876700024777344/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763840 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763818.871606:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2ed80 x1876700024777344/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763840 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763818.871664:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763818.871680:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763818.871688:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024777344:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789763818.871703:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763815/real 1789763818] req@ffffa06c08e2c700 x1876700024777088/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763840 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763818.871731:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2c700 x1876700024777088/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763840 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763818.871747:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763818.871754:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763818.871761:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024777088:192.168.203.107@tcp:250:kworker.0 00010000:00010000:3.0:1789763819.127105:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763819.382999:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789763820.151155:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763820.407052:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789763820.918993:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763820.919005:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763820.919009:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763820.919021:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 3254 00000100:00080000:2.0:1789763820.919034:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763820.919052:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789763820.919091:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e53480 x1876700024777472/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763820.919129:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763820.919136:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763820.919139:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763820.919144:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 3254 00000100:00080000:2.0:1789763820.919150:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763820.919167:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789763820.919183:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763820.919187:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e53480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024777472:192.168.203.107@tcp:250:kworker.0 00000100:00100000:2.0:1789763820.919217:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50a80 x1876700024777600/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763820.919251:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763820.919256:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763820.919258:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763820.919266:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 3254 00000800:00000100:1.0:1789763820.919272:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00080000:2.0:1789763820.919273:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763820.919281:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789763820.919288:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763820.919304:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50700 x1876700024777728/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763820.919332:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763820/real 1789763820] req@ffffa06c07e53480 x1876700024777472/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763845 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763820.919354:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e53480 x1876700024777472/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763845 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763820.919366:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763820.919371:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789763820.919375:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e53480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024777472:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789763820.919400:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024777600:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789763820.919425:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789763820.919440:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763820/real 1789763820] req@ffffa06c07e50a80 x1876700024777600/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763845 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763820.919453:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50a80 x1876700024777600/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763845 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763820.919461:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763820.919464:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789763820.919476:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024777600:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789763820.919483:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024777728:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789763820.919515:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789763820.919545:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763820/real 1789763820] req@ffffa06c07e50700 x1876700024777728/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763845 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763820.919558:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50700 x1876700024777728/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763845 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763820.919567:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763820.919571:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789763820.919574:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024777728:192.168.203.107@tcp:38:kworker.0 00010000:00010000:0.0:1789763821.174954:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763821.430930:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789763822.198980:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763822.454933:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763823.222942:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763823.478918:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789763824.247000:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763824.502965:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789763825.271002:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763825.526916:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789763826.038915:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763826.038925:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763826.038930:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763826.038941:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 3259 00000100:00080000:2.0:1789763826.038954:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763826.038962:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789763826.039002:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e53100 x1876700024777856/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763826.039025:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763826.039028:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763826.039030:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763826.039033:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 3259 00000100:00080000:2.0:1789763826.039049:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763826.039053:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789763826.039061:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763826.039062:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e53100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024777856:192.168.203.107@tcp:250:kworker.0 00000100:00100000:2.0:1789763826.039074:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e52a00 x1876700024777984/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763826.039092:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763826.039095:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763826.039096:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763826.039099:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 3259 00000100:00080000:2.0:1789763826.039101:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763826.039104:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789763826.039108:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1789763826.039115:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789763826.039118:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e52d80 x1876700024778112/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763826.039152:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763826/real 1789763826] req@ffffa06c07e53100 x1876700024777856/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763851 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763826.039180:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e53100 x1876700024777856/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763851 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763826.039188:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763826.039191:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789763826.039194:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e53100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024777856:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789763826.039211:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e52a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024777984:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789763826.039226:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789763826.039244:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763826/real 1789763826] req@ffffa06c07e52a00 x1876700024777984/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763851 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763826.039253:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e52a00 x1876700024777984/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763851 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763826.039259:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763826.039261:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789763826.039263:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e52a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024777984:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789763826.039267:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e52d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024778112:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789763826.039279:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789763826.039285:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763826/real 1789763826] req@ffffa06c07e52d80 x1876700024778112/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763851 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763826.039293:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e52d80 x1876700024778112/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763851 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763826.039297:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763826.039299:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789763826.039301:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e52d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024778112:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789763826.295020:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763826.550937:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789763827.318955:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763827.574914:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789763828.343035:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763828.598928:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763829.367029:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763829.622914:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763830.391012:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763830.646922:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789763831.159055:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763831.159067:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763831.159072:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763831.159084:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 3264 00000100:00080000:0.0:1789763831.159135:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763831.159142:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789763831.159175:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d2680 x1876700024778240/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763831.159195:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763831.159198:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763831.159200:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763831.159203:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 3264 00000100:00080000:0.0:1789763831.159205:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763831.159207:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789763831.159214:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763831.159243:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024778240:192.168.203.107@tcp:250:kworker.0 00000100:00100000:0.0:1789763831.159244:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0380 x1876700024778368/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763831.159258:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763831.159261:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763831.159262:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763831.159265:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 3264 00000100:00080000:0.0:1789763831.159268:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763831.159270:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789763831.159274:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763831.159292:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d3100 x1876700024778496/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789763831.159325:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789763831.159368:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763831/real 1789763831] req@ffffa06c091d2680 x1876700024778240/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763856 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763831.159396:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d2680 x1876700024778240/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763856 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763831.159435:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763831.159441:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789763831.159446:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024778240:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789763831.159488:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024778368:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789763831.159523:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789763831.159540:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763831/real 1789763831] req@ffffa06c091d0380 x1876700024778368/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763856 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763831.159556:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0380 x1876700024778368/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763856 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763831.159566:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763831.159572:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789763831.159579:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024778368:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789763831.159603:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024778496:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789763831.159623:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789763831.159634:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763831/real 1789763831] req@ffffa06c091d3100 x1876700024778496/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763856 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763831.159647:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d3100 x1876700024778496/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763856 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763831.159655:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763831.159657:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789763831.159659:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024778496:192.168.203.107@tcp:38:kworker.0 00010000:00010000:0.0:1789763831.415136:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763831.670983:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763832.438950:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763832.694927:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763833.463024:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763833.718957:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789763834.486937:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763834.742916:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763835.511113:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763835.767055:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789763836.279303:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763836.279313:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763836.279317:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763836.279328:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 3270 00000100:00080000:2.0:1789763836.279340:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763836.279348:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789763836.279412:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e52d80 x1876700024778624/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763836.279453:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763836.279479:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763836.279483:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763836.279492:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 3270 00000100:00080000:2.0:1789763836.279500:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763836.279507:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789763836.279522:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e52d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024778624:192.168.203.107@tcp:250:kworker.0 00000008:00100000:2.0:1789763836.279524:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763836.279565:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e52a00 x1876700024778752/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763836.279589:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763836.279593:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763836.279595:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763836.279601:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 3270 00000100:00080000:2.0:1789763836.279618:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763836.279622:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789763836.279629:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763836.279641:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e53100 x1876700024778880/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789763836.279670:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789763836.279759:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763836/real 1789763836] req@ffffa06c07e52d80 x1876700024778624/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763861 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763836.279797:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e52d80 x1876700024778624/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763861 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763836.279821:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763836.279831:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789763836.279856:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e52d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024778624:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789763836.279955:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e52a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024778752:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789763836.280004:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789763836.280023:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763836/real 1789763836] req@ffffa06c07e52a00 x1876700024778752/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763861 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763836.280039:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e52a00 x1876700024778752/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763861 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763836.280051:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763836.280055:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789763836.280060:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e52a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024778752:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789763836.280068:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e53100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024778880:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789763836.280116:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789763836.280133:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763836/real 1789763836] req@ffffa06c07e53100 x1876700024778880/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763861 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763836.280189:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e53100 x1876700024778880/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763861 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763836.280199:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763836.280203:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789763836.280206:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e53100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024778880:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789763836.535123:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763836.791136:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763837.559036:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763837.815022:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763838.583102:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763838.839099:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763839.607142:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763839.863011:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763840.631081:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763840.887000:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789763841.399028:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763841.399046:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763841.399074:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763841.399087:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 3275 00000100:00080000:0.0:1789763841.399101:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763841.399119:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789763841.399163:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d1f80 x1876700024779008/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763841.399196:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763841.399201:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763841.399204:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763841.399228:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 3275 00000100:00080000:0.0:1789763841.399235:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763841.399242:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789763841.399255:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763841.399273:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024779008:192.168.203.107@tcp:250:kworker.0 00000100:00100000:0.0:1789763841.399276:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0a80 x1876700024779136/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763841.399300:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763841.399305:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763841.399308:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763841.399312:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 3275 00000100:00080000:0.0:1789763841.399317:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763841.399322:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789763841.399328:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763841.399343:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0700 x1876700024779264/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789763841.399368:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789763841.399422:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763841/real 1789763841] req@ffffa06c091d1f80 x1876700024779008/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763866 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763841.399446:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d1f80 x1876700024779008/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763866 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763841.399467:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763841.399474:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789763841.399482:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024779008:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789763841.399522:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024779136:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789763841.399580:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789763841.399600:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763841/real 1789763841] req@ffffa06c091d0a80 x1876700024779136/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763866 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763841.399616:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0a80 x1876700024779136/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763866 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763841.399660:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763841.399665:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789763841.399670:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024779136:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789763841.399679:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024779264:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789763841.399708:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789763841.399722:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763841/real 1789763841] req@ffffa06c091d0700 x1876700024779264/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763866 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763841.399736:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0700 x1876700024779264/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763866 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763841.399746:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763841.399750:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789763841.399754:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024779264:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789763841.655016:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763841.910953:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763842.679022:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763842.935129:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763843.703017:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763843.958960:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763844.726957:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763844.982923:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763845.751069:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763846.006963:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:1.0:1789763846.519034:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763846.519075:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789763846.519080:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763846.519091:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 3280 00000100:00080000:1.0:1789763846.519103:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763846.519111:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789763846.519187:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2d180 x1876700024779392/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763846.519243:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763846.519273:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763846.519277:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763846.519286:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 3280 00000100:00080000:1.0:1789763846.519292:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763846.519300:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:3.0:1789763846.519314:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024779392:192.168.203.107@tcp:250:kworker.0 00000008:00100000:1.0:1789763846.519332:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763846.519360:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2c700 x1876700024779520/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763846.519382:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763846.519388:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763846.519393:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763846.519398:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 3280 00000100:00080000:1.0:1789763846.519405:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1789763846.519410:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00080000:1.0:1789763846.519420:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789763846.519445:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763846.519466:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2ed80 x1876700024779648/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763846.519503:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763846/real 1789763846] req@ffffa06c08e2d180 x1876700024779392/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763871 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763846.519536:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2d180 x1876700024779392/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763871 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763846.519548:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763846.519553:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763846.519558:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024779392:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789763846.519598:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024779520:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763846.519622:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763846.519634:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763846/real 1789763846] req@ffffa06c08e2c700 x1876700024779520/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763871 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763846.519671:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2c700 x1876700024779520/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763871 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763846.519678:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763846.519681:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763846.519683:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024779520:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789763846.519688:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024779648:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763846.519701:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763846.519708:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763846/real 1789763846] req@ffffa06c08e2ed80 x1876700024779648/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763871 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763846.519716:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2ed80 x1876700024779648/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763871 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763846.519742:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763846.519745:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763846.519748:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024779648:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789763846.775048:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763847.030937:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763847.799113:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763848.054967:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789763848.822985:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763849.078944:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763849.847048:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763850.102915:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763850.871083:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763851.126968:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789763851.639054:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763851.639072:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763851.639079:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763851.639096:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 3285 00000100:00080000:2.0:1789763851.639114:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763851.639124:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789763851.639191:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50700 x1876700024779776/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763851.639234:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763851.639239:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763851.639242:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763851.639247:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 3285 00000100:00080000:2.0:1789763851.639253:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763851.639258:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789763851.639269:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763851.639287:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50a80 x1876700024779904/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763851.639304:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763851.639309:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763851.639312:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763851.639317:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 3285 00000100:00080000:2.0:1789763851.639321:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763851.639325:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789763851.639332:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763851.639340:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024779776:192.168.203.107@tcp:250:kworker.0 00000100:00100000:2.0:1789763851.639350:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e53480 x1876700024780032/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789763851.639464:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763851.639531:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763851/real 1789763851] req@ffffa06c07e50700 x1876700024779776/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763876 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763851.639563:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50700 x1876700024779776/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763876 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763851.639582:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763851.639590:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763851.639598:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024779776:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789763851.639613:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024779904:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763851.639651:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763851.639690:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763851/real 1789763851] req@ffffa06c07e50a80 x1876700024779904/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763876 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763851.639713:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50a80 x1876700024779904/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763876 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763851.639729:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763851.639736:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763851.639742:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024779904:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789763851.639778:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e53480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024780032:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763851.639813:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763851.639832:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763851/real 1789763851] req@ffffa06c07e53480 x1876700024780032/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763876 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763851.639915:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e53480 x1876700024780032/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763876 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763851.639932:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763851.639938:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763851.639944:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e53480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024780032:192.168.203.107@tcp:38:kworker.0 00010000:00010000:2.0:1789763851.895089:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763852.151020:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789763852.919171:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763853.175011:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763853.943019:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763854.199085:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789763854.967120:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763855.222999:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789763855.991075:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763856.246989:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789763856.758991:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763856.759005:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763856.759010:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763856.759025:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 3290 00000100:00080000:2.0:1789763856.759052:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763856.759065:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789763856.759117:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51c00 x1876700024780160/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763856.759169:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763856.759174:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763856.759177:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763856.759184:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 3290 00000100:00080000:2.0:1789763856.759191:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763856.759198:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789763856.759212:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763856.759235:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024780160:192.168.203.107@tcp:250:kworker.0 00000100:00100000:2.0:1789763856.759238:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e53800 x1876700024780288/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763856.759271:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763856.759277:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763856.759280:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763856.759288:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 3290 00000100:00080000:2.0:1789763856.759295:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763856.759303:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789763856.759312:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1789763856.759342:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789763856.759357:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e52680 x1876700024780416/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789763856.759419:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763856/real 1789763856] req@ffffa06c07e51c00 x1876700024780160/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763881 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763856.759457:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51c00 x1876700024780160/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763881 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763856.759481:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763856.759490:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789763856.759512:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024780160:192.168.203.107@tcp:250:kworker.0 00000100:00100000:0.0:1789763856.759553:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e53800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024780288:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789763856.759603:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789763856.759628:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763856/real 1789763856] req@ffffa06c07e53800 x1876700024780288/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763881 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763856.759645:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e53800 x1876700024780288/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763881 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763856.759656:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763856.759660:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789763856.759664:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e53800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024780288:192.168.203.107@tcp:38:kworker.0 00000100:00100000:0.0:1789763856.759674:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e52680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024780416:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789763856.759703:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789763856.759716:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763856/real 1789763856] req@ffffa06c07e52680 x1876700024780416/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763881 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763856.759731:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e52680 x1876700024780416/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763881 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763856.759741:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763856.759745:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789763856.759749:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e52680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024780416:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789763857.015048:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763857.270998:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763858.038941:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763858.294946:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763859.062947:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763859.318994:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763860.087070:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763860.343058:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763861.111080:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763861.367032:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789763861.879008:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763861.879025:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763861.879051:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763861.879066:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 3295 00000100:00080000:0.0:1789763861.879082:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763861.879092:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789763861.879147:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0700 x1876700024780544/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763861.879185:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763861.879189:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763861.879192:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763861.879205:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 3295 00000100:00080000:0.0:1789763861.879209:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763861.879215:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789763861.879228:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763861.879238:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024780544:192.168.203.107@tcp:250:kworker.0 00000100:00100000:0.0:1789763861.879245:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0a80 x1876700024780672/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763861.879260:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763861.879264:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763861.879266:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763861.879270:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 3295 00000100:00080000:0.0:1789763861.879274:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763861.879279:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789763861.879286:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763861.879300:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d1f80 x1876700024780800/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789763861.879331:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763861.879388:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763861/real 1789763861] req@ffffa06c091d0700 x1876700024780544/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763886 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763861.879403:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0700 x1876700024780544/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763886 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763861.879412:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763861.879415:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763861.879419:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024780544:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789763861.879443:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024780672:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763861.879459:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763861.879468:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763861/real 1789763861] req@ffffa06c091d0a80 x1876700024780672/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763886 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763861.879476:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0a80 x1876700024780672/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763886 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763861.879482:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763861.879485:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763861.879495:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024780672:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789763861.879499:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024780800:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763861.879515:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763861.879522:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763861/real 1789763861] req@ffffa06c091d1f80 x1876700024780800/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763886 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763861.879536:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d1f80 x1876700024780800/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763886 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763861.879542:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763861.879544:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763861.879547:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024780800:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789763862.135065:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763862.390986:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763863.159075:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763863.414976:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763864.183094:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763864.438958:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789763865.207007:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763865.462953:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763866.231033:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763866.486966:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789763866.999027:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763866.999040:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763866.999046:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763866.999060:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 3300 00000100:00080000:0.0:1789763866.999074:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763866.999083:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789763866.999145:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d3100 x1876700024780928/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763866.999183:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763866.999188:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763866.999191:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763866.999195:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 3300 00000100:00080000:0.0:1789763866.999200:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763866.999204:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789763866.999216:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763866.999230:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0380 x1876700024781056/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763866.999244:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763866.999248:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763866.999250:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763866.999254:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 3300 00000100:00080000:0.0:1789763866.999257:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763866.999261:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789763866.999288:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763866.999303:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d2680 x1876700024781184/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763866.999320:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024780928:192.168.203.107@tcp:250:kworker.0 00000800:00000100:3.0:1789763866.999413:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763866.999480:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763866/real 1789763866] req@ffffa06c091d3100 x1876700024780928/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763891 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763866.999505:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d3100 x1876700024780928/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763891 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763866.999531:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763866.999537:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763866.999543:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024780928:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789763866.999552:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024781056:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763866.999582:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763866.999598:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763866/real 1789763866] req@ffffa06c091d0380 x1876700024781056/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763891 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763866.999614:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0380 x1876700024781056/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763891 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763866.999626:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763866.999632:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763866.999638:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024781056:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789763866.999678:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024781184:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763866.999717:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763866.999741:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763866/real 1789763866] req@ffffa06c091d2680 x1876700024781184/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763891 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763866.999766:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d2680 x1876700024781184/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763891 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763866.999778:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763866.999782:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763866.999786:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024781184:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789763867.255056:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763867.510998:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763868.279046:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763868.534947:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763869.303062:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763869.558922:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789763870.327102:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763870.582998:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789763871.351053:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763871.606944:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789763872.119052:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763872.119064:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763872.119070:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763872.119083:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 3305 00000100:00080000:2.0:1789763872.119097:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763872.119106:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789763872.119155:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50380 x1876700024781312/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763872.119193:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763872.119198:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763872.119200:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763872.119206:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 3305 00000100:00080000:2.0:1789763872.119210:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763872.119214:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789763872.119225:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763872.119240:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e53b80 x1876700024781440/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763872.119254:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763872.119258:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763872.119260:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763872.119264:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 3305 00000100:00080000:2.0:1789763872.119290:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763872.119297:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789763872.119305:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763872.119323:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024781312:192.168.203.107@tcp:250:kworker.0 00000100:00100000:2.0:1789763872.119325:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51f80 x1876700024781568/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789763872.119421:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763872.119475:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763872/real 1789763872] req@ffffa06c07e50380 x1876700024781312/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763897 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763872.119498:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50380 x1876700024781312/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763897 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763872.119513:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763872.119518:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763872.119523:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024781312:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789763872.119533:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e53b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024781440:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763872.119557:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763872.119571:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763872/real 1789763872] req@ffffa06c07e53b80 x1876700024781440/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763897 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763872.119586:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e53b80 x1876700024781440/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763897 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763872.119595:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763872.119612:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763872.119617:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e53b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024781440:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789763872.119647:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024781568:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763872.119674:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763872.119688:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763872/real 1789763872] req@ffffa06c07e51f80 x1876700024781568/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763897 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763872.119703:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51f80 x1876700024781568/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763897 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763872.119713:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763872.119717:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763872.119721:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024781568:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789763872.375047:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763872.630965:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789763873.399075:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763873.654984:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789763874.423065:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763874.678987:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763875.447032:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763875.702951:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789763876.471167:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763876.726947:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789763877.238998:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763877.239012:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763877.239017:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763877.239031:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 3311 00000100:00080000:0.0:1789763877.239044:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763877.239054:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789763877.239102:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d2a00 x1876700024781696/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763877.239141:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763877.239146:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763877.239148:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763877.239154:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 3311 00000100:00080000:0.0:1789763877.239158:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763877.239163:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789763877.239175:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763877.239192:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0e00 x1876700024781824/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763877.239208:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763877.239213:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:3.0:1789763877.239230:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024781696:192.168.203.107@tcp:250:kworker.0 00000100:00080000:0.0:1789763877.239254:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763877.239266:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 3311 00000100:00080000:0.0:1789763877.239275:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763877.239285:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789763877.239297:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763877.239325:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d1500 x1876700024781952/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789763877.239339:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763877.239419:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763877/real 1789763877] req@ffffa06c091d2a00 x1876700024781696/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763902 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763877.239458:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d2a00 x1876700024781696/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763902 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763877.239494:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763877.239500:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763877.239506:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024781696:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789763877.239535:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024781824:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763877.239573:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763877.239593:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763877/real 1789763877] req@ffffa06c091d0e00 x1876700024781824/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763902 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763877.239609:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0e00 x1876700024781824/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763902 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763877.239619:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763877.239624:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763877.239628:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024781824:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789763877.239657:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024781952:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763877.239685:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763877.239697:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763877/real 1789763877] req@ffffa06c091d1500 x1876700024781952/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763902 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763877.239712:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d1500 x1876700024781952/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763902 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763877.239722:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763877.239726:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763877.239729:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024781952:192.168.203.107@tcp:38:kworker.0 00010000:00010000:0.0:1789763877.494967:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763877.751042:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789763878.518992:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763878.774964:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789763879.543089:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763879.799005:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789763880.566986:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763880.822957:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789763881.590988:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763881.846919:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:1.0:1789763882.358972:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763882.358999:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789763882.359002:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763882.359010:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 3316 00000100:00080000:1.0:1789763882.359032:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763882.359038:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789763882.359106:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2ca80 x1876700024782080/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763882.359139:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763882.359144:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763882.359147:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763882.359152:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 3316 00000100:00080000:1.0:1789763882.359157:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763882.359162:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789763882.359201:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763882.359219:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2c000 x1876700024782208/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763882.359228:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024782080:192.168.203.107@tcp:250:kworker.0 00000100:00080000:1.0:1789763882.359256:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763882.359260:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763882.359262:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763882.359265:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 3316 00000100:00080000:1.0:1789763882.359267:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763882.359270:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789763882.359274:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763882.359284:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2e300 x1876700024782336/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763882.359356:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024782208:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789763882.359385:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024782336:192.168.203.107@tcp:38:kworker.0 00010000:00010000:1.0:1789763882.615083:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763882.870955:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789763883.639070:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763883.894971:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789763884.663041:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763884.918993:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000400:00000100:2.0:1789763885.431237:0:2328:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.107:988 00000400:00000100:2.0:1789763885.431314:0:2328:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.107@tcp at host 192.168.203.107 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1789763885.431346:0:2328:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.107@tcp 00000800:00000100:2.0:1789763885.431432:0:2328:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.107@tcp 00000800:00000100:2.0:1789763885.431454:0:2328:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.107@tcp 00000100:00100000:3.0:1789763885.431485:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763882/real 1789763885] req@ffffa06c08e2c000 x1876700024782208/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763907 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763885.431531:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2c000 x1876700024782208/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763907 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763885.431555:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763885.431576:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763885.431584:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024782208:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789763885.431600:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763882/real 1789763885] req@ffffa06c08e2e300 x1876700024782336/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763907 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763885.431626:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2e300 x1876700024782336/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763907 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763885.431643:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763885.431666:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763885.431674:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024782336:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789763885.431687:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763882/real 1789763885] req@ffffa06c08e2ca80 x1876700024782080/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763907 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763885.431714:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2ca80 x1876700024782080/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763907 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763885.431755:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763885.431762:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763885.431770:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024782080:192.168.203.107@tcp:250:kworker.0 00010000:00010000:2.0:1789763885.687093:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763885.942975:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789763886.711037:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763886.966960:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789763887.478998:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763887.479013:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763887.479018:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763887.479049:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 3321 00000100:00080000:0.0:1789763887.479063:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763887.479071:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789763887.479147:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0000 x1876700024782464/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763887.479193:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763887.479203:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763887.479206:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763887.479241:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 3321 00000100:00100000:3.0:1789763887.479242:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024782464:192.168.203.107@tcp:250:kworker.0 00000100:00080000:0.0:1789763887.479252:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763887.479276:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789763887.479306:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763887.479329:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d3b80 x1876700024782592/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763887.479346:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763887.479352:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000800:00000100:3.0:1789763887.479359:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00080000:0.0:1789763887.479360:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763887.479371:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 3321 00000100:00080000:0.0:1789763887.479378:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763887.479386:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789763887.479393:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763887.479414:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d2300 x1876700024782720/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763887.479419:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763887/real 1789763887] req@ffffa06c091d0000 x1876700024782464/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763912 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763887.479459:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0000 x1876700024782464/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763912 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763887.479474:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763887.479480:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763887.479486:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024782464:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789763887.479525:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024782592:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763887.479554:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763887.479583:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763887/real 1789763887] req@ffffa06c091d3b80 x1876700024782592/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763912 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763887.479601:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d3b80 x1876700024782592/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763912 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763887.479621:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763887.479626:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763887.479631:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024782592:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789763887.479641:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024782720:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763887.479669:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763887.479687:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763887/real 1789763887] req@ffffa06c091d2300 x1876700024782720/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763912 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763887.479707:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d2300 x1876700024782720/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763912 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763887.479734:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763887.479740:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763887.479745:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024782720:192.168.203.107@tcp:38:kworker.0 00010000:00010000:0.0:1789763887.735064:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763887.990992:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789763888.759093:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763889.014938:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789763889.782964:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763890.038923:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789763890.807093:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763891.062933:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789763891.831036:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763892.086977:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789763892.599054:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763892.599068:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763892.599076:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763892.599105:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 3326 00000100:00080000:0.0:1789763892.599122:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763892.599136:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789763892.599204:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d2300 x1876700024782848/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763892.599245:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763892.599250:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763892.599266:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763892.599273:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 3326 00000100:00080000:0.0:1789763892.599280:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763892.599288:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789763892.599303:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763892.599311:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024782848:192.168.203.107@tcp:250:kworker.0 00000100:00100000:0.0:1789763892.599334:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d3b80 x1876700024782976/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763892.599355:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763892.599361:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763892.599364:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763892.599371:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 3326 00000100:00080000:0.0:1789763892.599378:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763892.599382:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789763892.599390:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763892.599423:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0000 x1876700024783104/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789763892.599431:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789763892.599496:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763892/real 1789763892] req@ffffa06c091d2300 x1876700024782848/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763917 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763892.599535:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d2300 x1876700024782848/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763917 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763892.599558:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763892.599567:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789763892.599575:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024782848:192.168.203.107@tcp:250:kworker.0 00000100:00100000:2.0:1789763892.599643:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024782976:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789763892.599724:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789763892.599749:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763892/real 1789763892] req@ffffa06c091d3b80 x1876700024782976/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763917 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763892.599771:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d3b80 x1876700024782976/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763917 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763892.599783:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763892.599790:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789763892.599799:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024782976:192.168.203.107@tcp:38:kworker.0 00000100:00100000:2.0:1789763892.599811:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024783104:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789763892.599909:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789763892.599954:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763892/real 1789763892] req@ffffa06c091d0000 x1876700024783104/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763917 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763892.599985:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0000 x1876700024783104/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763917 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763892.600003:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763892.600010:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789763892.600015:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024783104:192.168.203.107@tcp:38:kworker.0 00010000:00010000:0.0:1789763892.855085:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789763893.111033:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789763893.879043:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763894.135041:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789763894.903049:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763895.158964:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789763895.927031:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789763896.183040:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789763896.951104:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789763897.206965:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789763897.719019:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763897.719032:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763897.719062:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763897.719075:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 3331 00000100:00080000:2.0:1789763897.719089:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763897.719114:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789763897.719166:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50e00 x1876700024783232/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763897.719206:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763897.719211:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763897.719214:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763897.719219:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 3331 00000100:00080000:2.0:1789763897.719223:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763897.719228:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789763897.719240:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763897.719254:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51500 x1876700024783360/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763897.719268:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763897.719272:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763897.719274:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763897.719278:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 3331 00000100:00080000:2.0:1789763897.719281:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763897.719285:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789763897.719291:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763897.719302:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50000 x1876700024783488/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763897.719346:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024783232:192.168.203.107@tcp:250:kworker.0 00000800:00000100:3.0:1789763897.719465:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763897.719522:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763897/real 1789763897] req@ffffa06c07e50e00 x1876700024783232/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763922 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763897.719546:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50e00 x1876700024783232/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763922 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763897.719560:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763897.719566:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763897.719571:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024783232:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789763897.719581:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024783360:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763897.719608:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763897.719622:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763897/real 1789763897] req@ffffa06c07e51500 x1876700024783360/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763922 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763897.719663:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51500 x1876700024783360/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763922 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763897.719675:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763897.719679:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763897.719684:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024783360:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789763897.719713:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024783488:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763897.719741:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763897.719756:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763897/real 1789763897] req@ffffa06c07e50000 x1876700024783488/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763922 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763897.719791:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50000 x1876700024783488/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763922 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763897.719801:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763897.719805:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763897.719809:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024783488:192.168.203.107@tcp:38:kworker.0 00010000:00010000:0.0:1789763897.975108:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789763898.230990:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789763898.999151:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789763899.255034:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789763900.023075:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789763900.279033:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789763901.047044:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789763901.303025:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789763902.071072:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789763902.326995:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:1.0:1789763902.839044:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763902.839072:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789763902.839077:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763902.839088:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 3336 00000100:00080000:1.0:1789763902.839101:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763902.839120:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789763902.839173:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2d500 x1876700024783616/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763902.839217:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763902.839225:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763902.839228:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763902.839235:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 3336 00000100:00080000:1.0:1789763902.839242:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763902.839247:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789763902.839259:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763902.839277:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2ca80 x1876700024783744/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763902.839299:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763902.839304:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763902.839308:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1789763902.839319:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024783616:192.168.203.107@tcp:250:kworker.0 00000100:00080000:1.0:1789763902.839338:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 3336 00000100:00080000:1.0:1789763902.839348:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763902.839356:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789763902.839367:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763902.839388:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2e300 x1876700024783872/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789763902.839403:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763902.839454:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763902/real 1789763902] req@ffffa06c08e2d500 x1876700024783616/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763927 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763902.839475:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2d500 x1876700024783616/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763927 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763902.839487:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763902.839492:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763902.839497:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024783616:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789763902.839505:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024783744:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763902.839525:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763902.839538:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763902/real 1789763902] req@ffffa06c08e2ca80 x1876700024783744/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763927 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763902.839551:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2ca80 x1876700024783744/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763927 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763902.839577:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763902.839582:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763902.839586:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024783744:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789763902.839648:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024783872:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763902.839674:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763902.839687:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763902/real 1789763902] req@ffffa06c08e2e300 x1876700024783872/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763927 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763902.839724:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2e300 x1876700024783872/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763927 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763902.839734:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763902.839737:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763902.839741:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024783872:192.168.203.107@tcp:38:kworker.0 00010000:00010000:1.0:1789763903.095167:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789763903.350949:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789763904.119097:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763904.375098:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789763905.143038:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763905.398930:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789763906.167073:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763906.422938:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789763907.191045:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763907.446965:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789763907.958994:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763907.959006:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763907.959012:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763907.959024:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 3341 00000100:00080000:2.0:1789763907.959038:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763907.959048:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789763907.959110:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51180 x1876700024784000/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763907.959143:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763907.959148:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763907.959151:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763907.959155:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 3341 00000100:00080000:2.0:1789763907.959160:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763907.959165:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789763907.959177:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763907.959196:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e52300 x1876700024784128/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763907.959214:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763907.959219:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763907.959223:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1789763907.959229:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024784000:192.168.203.107@tcp:250:kworker.0 00000100:00080000:2.0:1789763907.959233:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 3341 00000100:00080000:2.0:1789763907.959258:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763907.959265:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789763907.959274:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763907.959293:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51880 x1876700024784256/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789763907.959331:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763907.959418:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763907/real 1789763907] req@ffffa06c07e51180 x1876700024784000/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763932 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763907.959445:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51180 x1876700024784000/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763932 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763907.959491:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763907.959497:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763907.959503:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024784000:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789763907.959533:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e52300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024784128:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763907.959599:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763907.959621:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763907/real 1789763907] req@ffffa06c07e52300 x1876700024784128/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763932 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763907.959637:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e52300 x1876700024784128/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763932 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763907.959649:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763907.959653:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763907.959657:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e52300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024784128:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789763907.959665:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024784256:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763907.959690:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763907.959702:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763907/real 1789763907] req@ffffa06c07e51880 x1876700024784256/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763932 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763907.959716:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51880 x1876700024784256/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763932 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763907.959726:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763907.959730:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763907.959734:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024784256:192.168.203.107@tcp:38:kworker.0 00010000:00010000:1.0:1789763908.215034:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763908.470971:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789763909.239067:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763909.494974:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789763910.263143:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763910.518962:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789763911.287153:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763911.542982:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789763912.311050:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763912.566967:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789763913.078986:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763913.079002:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763913.079009:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763913.079025:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 3346 00000100:00080000:0.0:1789763913.079043:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763913.079057:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789763913.079115:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d1500 x1876700024784384/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763913.079164:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763913.079170:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763913.079173:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763913.079178:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 3346 00000100:00080000:0.0:1789763913.079184:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763913.079189:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789763913.079202:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763913.079222:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0e00 x1876700024784512/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763913.079239:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1789763913.079263:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024784384:192.168.203.107@tcp:250:kworker.0 00000100:00080000:0.0:1789763913.079265:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763913.079273:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763913.079284:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 3346 00000100:00080000:0.0:1789763913.079290:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763913.079295:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789763913.079303:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763913.079326:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d2a00 x1876700024784640/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789763913.079363:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763913.079419:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763913/real 1789763913] req@ffffa06c091d1500 x1876700024784384/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763938 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763913.079442:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d1500 x1876700024784384/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763938 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763913.079458:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763913.079463:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763913.079469:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024784384:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789763913.079479:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024784512:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763913.079503:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763913.079516:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763913/real 1789763913] req@ffffa06c091d0e00 x1876700024784512/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763938 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763913.079565:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0e00 x1876700024784512/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763938 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763913.079577:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763913.079581:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763913.079585:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024784512:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789763913.079629:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024784640:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763913.079660:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763913.079675:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763913/real 1789763913] req@ffffa06c091d2a00 x1876700024784640/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763938 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763913.079690:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d2a00 x1876700024784640/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763938 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763913.079701:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763913.079705:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763913.079709:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024784640:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789763913.335009:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763913.590950:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789763914.359027:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763914.615010:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763915.383042:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763915.638952:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763916.407052:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763916.662948:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763917.431073:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763917.686999:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:1.0:1789763918.199094:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763918.199102:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789763918.199105:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763918.199114:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 3352 00000100:00080000:1.0:1789763918.199124:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763918.199130:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789763918.199183:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2c000 x1876700024784768/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763918.199224:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763918.199227:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763918.199229:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763918.199232:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 3352 00000100:00080000:1.0:1789763918.199235:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763918.199237:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789763918.199254:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763918.199291:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2ed80 x1876700024784896/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763918.199302:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1789763918.199306:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024784768:192.168.203.107@tcp:250:kworker.0 00000100:00080000:1.0:1789763918.199307:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763918.199310:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763918.199317:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 3352 00000100:00080000:1.0:1789763918.199320:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763918.199323:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789763918.199327:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763918.199352:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2c700 x1876700024785024/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789763918.199385:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763918.199443:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763918/real 1789763918] req@ffffa06c08e2c000 x1876700024784768/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763943 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763918.199460:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2c000 x1876700024784768/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763943 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763918.199478:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763918.199481:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763918.199497:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024784768:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789763918.199564:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024784896:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763918.199583:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763918.199593:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763918/real 1789763918] req@ffffa06c08e2ed80 x1876700024784896/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763943 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763918.199602:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2ed80 x1876700024784896/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763943 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763918.199607:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763918.199616:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763918.199619:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024784896:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789763918.199623:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024785024:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763918.199634:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763918.199641:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763918/real 1789763918] req@ffffa06c08e2c700 x1876700024785024/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763943 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763918.199648:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2c700 x1876700024785024/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763943 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763918.199654:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763918.199656:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763918.199658:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024785024:192.168.203.107@tcp:38:kworker.0 00010000:00010000:2.0:1789763918.454973:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763918.710978:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763919.479043:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763919.734952:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763920.503119:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763920.758979:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789763921.527037:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763921.783080:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763922.551091:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763922.806964:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789763923.319013:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763923.319029:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763923.319034:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763923.319046:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 3357 00000100:00080000:0.0:1789763923.319060:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763923.319069:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789763923.319114:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d2680 x1876700024785152/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763923.319148:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763923.319153:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763923.319156:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763923.319160:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 3357 00000100:00080000:0.0:1789763923.319164:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763923.319168:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789763923.319200:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763923.319229:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0380 x1876700024785280/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763923.319247:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1789763923.319248:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024785152:192.168.203.107@tcp:250:kworker.0 00000100:00080000:0.0:1789763923.319257:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763923.319265:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763923.319272:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 3357 00000100:00080000:0.0:1789763923.319277:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763923.319281:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789763923.319288:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763923.319301:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d3100 x1876700024785408/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789763923.319345:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763923.319398:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763923/real 1789763923] req@ffffa06c091d2680 x1876700024785152/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763948 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763923.319420:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d2680 x1876700024785152/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763948 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763923.319435:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763923.319440:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763923.319444:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024785152:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789763923.319507:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024785280:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763923.319536:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763923.319552:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763923/real 1789763923] req@ffffa06c091d0380 x1876700024785280/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763948 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763923.319567:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0380 x1876700024785280/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763948 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763923.319577:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763923.319591:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763923.319596:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024785280:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789763923.319604:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024785408:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763923.319629:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763923.319641:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763923/real 1789763923] req@ffffa06c091d3100 x1876700024785408/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763948 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763923.319655:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d3100 x1876700024785408/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763948 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763923.319664:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763923.319668:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763923.319672:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024785408:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789763923.575063:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763923.831077:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789763924.599034:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763924.854993:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763925.623023:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763925.878992:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763926.647036:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763926.902935:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789763927.671100:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763927.926975:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789763928.439011:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763928.439028:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763928.439035:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763928.439051:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 3362 00000100:00080000:0.0:1789763928.439068:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763928.439081:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789763928.439140:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d1f80 x1876700024785536/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763928.439186:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763928.439194:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763928.439218:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1789763928.439226:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024785536:192.168.203.107@tcp:250:kworker.0 00000100:00080000:0.0:1789763928.439228:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 3362 00000100:00080000:0.0:1789763928.439237:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763928.439246:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789763928.439262:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763928.439294:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0a80 x1876700024785664/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763928.439319:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:1.0:1789763928.439325:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00080000:0.0:1789763928.439327:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763928.439332:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763928.439340:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 3362 00000100:00080000:0.0:1789763928.439346:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763928.439352:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789763928.439361:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763928.439381:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0700 x1876700024785792/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763928.439391:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763928/real 1789763928] req@ffffa06c091d1f80 x1876700024785536/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763953 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763928.439450:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d1f80 x1876700024785536/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763953 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763928.439472:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763928.439480:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789763928.439488:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024785536:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789763928.439527:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024785664:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789763928.439591:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789763928.439613:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763928/real 1789763928] req@ffffa06c091d0a80 x1876700024785664/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763953 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763928.439634:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0a80 x1876700024785664/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763953 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763928.439646:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763928.439652:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789763928.439657:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024785664:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789763928.439665:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024785792:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789763928.439689:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789763928.439703:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763928/real 1789763928] req@ffffa06c091d0700 x1876700024785792/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763953 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763928.439717:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0700 x1876700024785792/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763953 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763928.439727:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763928.439731:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789763928.439735:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024785792:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789763928.695041:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763928.950993:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763929.719030:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763929.974956:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789763930.743088:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763930.998979:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763931.767087:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763932.022962:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763932.791094:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763933.047003:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789763933.559078:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763933.559091:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763933.559097:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763933.559110:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 3367 00000100:00080000:2.0:1789763933.559124:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763933.559134:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789763933.559186:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51880 x1876700024785920/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763933.559225:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763933.559230:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763933.559233:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763933.559238:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 3367 00000100:00080000:2.0:1789763933.559242:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763933.559246:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789763933.559258:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763933.559272:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e52300 x1876700024786048/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763933.559307:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763933.559312:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763933.559315:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763933.559322:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 3367 00000100:00080000:2.0:1789763933.559329:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763933.559338:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789763933.559349:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763933.559365:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024785920:192.168.203.107@tcp:250:kworker.0 00000100:00100000:2.0:1789763933.559400:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51180 x1876700024786176/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789763933.559499:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789763933.559559:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763933/real 1789763933] req@ffffa06c07e51880 x1876700024785920/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763958 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763933.559600:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51880 x1876700024785920/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763958 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763933.559616:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763933.559623:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789763933.559629:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024785920:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789763933.559639:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e52300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024786048:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789763933.559697:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789763933.559724:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763933/real 1789763933] req@ffffa06c07e52300 x1876700024786048/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763958 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763933.559740:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e52300 x1876700024786048/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763958 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763933.559751:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763933.559755:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789763933.559759:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e52300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024786048:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789763933.559788:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024786176:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789763933.559811:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789763933.559823:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763933/real 1789763933] req@ffffa06c07e51180 x1876700024786176/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763958 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763933.559838:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51180 x1876700024786176/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763958 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763933.559928:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763933.559936:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789763933.559943:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024786176:192.168.203.107@tcp:38:kworker.0 00010000:00010000:1.0:1789763933.815051:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763934.070968:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763934.838998:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763935.095033:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763935.863027:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763936.118993:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789763936.887125:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763937.142999:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763937.911064:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763938.166944:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:1.0:1789763938.679127:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763938.679140:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789763938.679146:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763938.679158:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 3372 00000100:00080000:1.0:1789763938.679172:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763938.679181:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789763938.679228:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2d180 x1876700024786304/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763938.679271:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763938.679276:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763938.679279:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763938.679284:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 3372 00000100:00080000:1.0:1789763938.679288:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763938.679293:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789763938.679304:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763938.679324:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2c700 x1876700024786432/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763938.679372:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1789763938.679388:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024786304:192.168.203.107@tcp:250:kworker.0 00000100:00080000:1.0:1789763938.679410:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763938.679414:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763938.679422:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 3372 00000100:00080000:1.0:1789763938.679429:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763938.679435:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789763938.679444:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763938.679465:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2ed80 x1876700024786560/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789763938.679499:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763938.679559:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763938/real 1789763938] req@ffffa06c08e2d180 x1876700024786304/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763963 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763938.679583:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2d180 x1876700024786304/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763963 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763938.679609:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763938.679616:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763938.679624:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024786304:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789763938.679664:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024786432:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763938.679718:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763938.679757:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763938/real 1789763938] req@ffffa06c08e2c700 x1876700024786432/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763963 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763938.679775:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2c700 x1876700024786432/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763963 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763938.679823:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763938.679828:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763938.679833:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024786432:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789763938.679891:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024786560:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763938.679942:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763938.679965:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763938/real 1789763938] req@ffffa06c08e2ed80 x1876700024786560/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763963 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763938.679981:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2ed80 x1876700024786560/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763963 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763938.679992:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763938.679996:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763938.680000:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024786560:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789763938.935033:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763939.190976:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789763939.959179:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763940.215022:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763940.983089:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763941.238995:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763942.007026:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763942.263023:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763943.031039:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763943.287009:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789763943.798993:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763943.799002:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763943.799006:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763943.799015:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 3377 00000100:00080000:0.0:1789763943.799026:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763943.799032:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789763943.799070:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0700 x1876700024786688/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763943.799112:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763943.799115:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763943.799117:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763943.799120:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 3377 00000100:00080000:0.0:1789763943.799123:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763943.799126:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789763943.799134:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763943.799145:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0a80 x1876700024786816/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763943.799154:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763943.799156:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763943.799158:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763943.799161:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 3377 00000100:00080000:0.0:1789763943.799164:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763943.799168:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789763943.799171:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763943.799178:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d1f80 x1876700024786944/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763943.799195:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024786688:192.168.203.107@tcp:250:kworker.0 00000800:00000100:3.0:1789763943.799256:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763943.799318:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763943/real 1789763943] req@ffffa06c091d0700 x1876700024786688/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763968 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763943.799335:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0700 x1876700024786688/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763968 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763943.799344:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763943.799348:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763943.799352:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024786688:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789763943.799358:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024786816:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763943.799377:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763943.799386:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763943/real 1789763943] req@ffffa06c091d0a80 x1876700024786816/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763968 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763943.799395:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0a80 x1876700024786816/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763968 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763943.799402:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763943.799404:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763943.799406:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024786816:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789763943.799425:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024786944:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763943.799438:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763943.799445:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763943/real 1789763943] req@ffffa06c091d1f80 x1876700024786944/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763968 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763943.799454:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d1f80 x1876700024786944/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763968 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763943.799471:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763943.799473:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763943.799476:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024786944:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789763944.055036:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763944.310982:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763945.079139:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763945.335013:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763946.103040:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763946.358992:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763947.127029:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763947.382926:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763948.150973:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763948.406905:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789763948.918950:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763948.918958:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763948.918961:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763948.918970:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 3382 00000100:00080000:2.0:1789763948.918979:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763948.918985:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789763948.919023:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50000 x1876700024787072/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763948.919044:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763948.919047:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763948.919049:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763948.919052:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 3382 00000100:00080000:2.0:1789763948.919066:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763948.919070:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789763948.919079:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763948.919101:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024787072:192.168.203.107@tcp:250:kworker.0 00000100:00100000:2.0:1789763948.919111:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51500 x1876700024787200/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763948.919123:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763948.919127:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763948.919129:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763948.919132:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 3382 00000100:00080000:2.0:1789763948.919135:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763948.919138:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789763948.919143:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763948.919153:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50e00 x1876700024787328/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763948.919221:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024787200:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789763948.919257:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024787328:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789763949.174977:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763949.430982:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763950.199007:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763950.454967:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763951.222977:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763951.478963:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000400:00000100:1.0:1789763951.991134:0:2329:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.107:988 00000400:00000100:1.0:1789763951.991177:0:2329:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.107@tcp at host 192.168.203.107 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1789763951.991192:0:2329:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.107@tcp 00000800:00000100:1.0:1789763951.991289:0:2329:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.107@tcp 00000800:00000100:1.0:1789763951.991300:0:2329:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.107@tcp 00000100:00100000:3.0:1789763951.991317:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763948/real 1789763951] req@ffffa06c07e51500 x1876700024787200/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763973 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763951.991339:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51500 x1876700024787200/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763973 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763951.991358:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763951.991362:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763951.991366:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024787200:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789763951.991372:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763948/real 1789763951] req@ffffa06c07e50e00 x1876700024787328/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763973 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763951.991381:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50e00 x1876700024787328/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763973 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763951.991390:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763951.991395:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763951.991399:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024787328:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789763951.991407:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763948/real 1789763951] req@ffffa06c07e50000 x1876700024787072/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763973 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763951.991422:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50000 x1876700024787072/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763973 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763951.991437:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763951.991441:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763951.991446:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024787072:192.168.203.107@tcp:250:kworker.0 00010000:00010000:3.0:1789763952.247043:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763952.503069:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763953.270987:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763953.526924:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789763954.038966:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763954.038995:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763954.039006:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763954.039020:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 3387 00000100:00080000:2.0:1789763954.039035:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763954.039046:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789763954.039093:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51f80 x1876700024787456/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763954.039154:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763954.039176:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763954.039181:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763954.039192:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 3387 00000100:00080000:2.0:1789763954.039202:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763954.039211:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:3.0:1789763954.039260:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024787456:192.168.203.107@tcp:250:kworker.0 00000008:00100000:2.0:1789763954.039271:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763954.039307:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e53b80 x1876700024787584/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763954.039330:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763954.039337:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763954.039340:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763954.039348:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 3387 00000100:00080000:2.0:1789763954.039355:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1789763954.039365:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00080000:2.0:1789763954.039367:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789763954.039380:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763954.039402:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50380 x1876700024787712/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763954.039417:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763954/real 1789763954] req@ffffa06c07e51f80 x1876700024787456/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763979 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763954.039444:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51f80 x1876700024787456/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763979 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763954.039461:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763954.039468:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763954.039475:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024787456:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789763954.039508:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e53b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024787584:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763954.039548:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763954.039569:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763954/real 1789763954] req@ffffa06c07e53b80 x1876700024787584/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763979 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763954.039600:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e53b80 x1876700024787584/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763979 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763954.039614:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763954.039620:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763954.039627:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e53b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024787584:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789763954.039637:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024787712:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789763954.039670:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789763954.039687:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763954/real 1789763954] req@ffffa06c07e50380 x1876700024787712/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763979 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763954.039733:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50380 x1876700024787712/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763979 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763954.039745:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763954.039750:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789763954.039755:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024787712:192.168.203.107@tcp:38:kworker.0 00010000:00010000:2.0:1789763954.295005:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763954.550971:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763955.318969:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763955.574908:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763956.343038:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763956.598961:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789763957.366970:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763957.622912:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763958.391012:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763958.646945:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:3.0:1789763959.158997:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763959.159007:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763959.159011:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763959.159021:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 3392 00000100:00080000:3.0:1789763959.159033:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763959.159040:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:3.0:1789763959.159114:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872f800 x1876700024787840/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763959.159165:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763959.159171:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763959.159173:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763959.159251:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 3392 00000100:00080000:3.0:1789763959.159256:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763959.159260:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789763959.159271:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763959.159319:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024787840:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789763959.159321:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872df80 x1876700024787968/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763959.159352:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763959.159356:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763959.159359:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763959.159365:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 3392 00000100:00080000:3.0:1789763959.159370:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763959.159381:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789763959.159389:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763959.159407:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872d880 x1876700024788096/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789763959.159446:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789763959.159499:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763959/real 1789763959] req@ffffa06c1872f800 x1876700024787840/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763984 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763959.159535:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872f800 x1876700024787840/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763984 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763959.159545:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763959.159548:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789763959.159552:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024787840:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789763959.159574:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024787968:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789763959.159603:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789763959.159614:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763959/real 1789763959] req@ffffa06c1872df80 x1876700024787968/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763984 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763959.159624:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872df80 x1876700024787968/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763984 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763959.159632:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763959.159634:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789763959.159637:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024787968:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789763959.159642:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024788096:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789763959.159655:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789763959.159663:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763959/real 1789763959] req@ffffa06c1872d880 x1876700024788096/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763984 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763959.159672:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872d880 x1876700024788096/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763984 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763959.159679:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763959.159681:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789763959.159683:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024788096:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789763959.415017:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763959.670952:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789763960.439034:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763960.694914:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789763961.462969:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763961.719000:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763962.487030:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763962.742980:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789763963.511078:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763963.766990:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:3.0:1789763964.278979:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763964.278993:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763964.278998:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763964.279034:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 3398 00000100:00080000:3.0:1789763964.279050:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763964.279061:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:3.0:1789763964.279144:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872c380 x1876700024788224/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763964.279231:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763964.279237:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763964.279240:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763964.279248:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 3398 00000100:00080000:3.0:1789763964.279253:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763964.279259:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789763964.279326:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763964.279331:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024788224:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789763964.279361:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872ce00 x1876700024788352/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763964.279407:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763964.279414:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763964.279417:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763964.279425:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 3398 00000100:00080000:3.0:1789763964.279433:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763964.279442:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789763964.279452:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763964.279476:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872e300 x1876700024788480/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789763964.279499:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789763964.279585:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763964/real 1789763964] req@ffffa06c1872c380 x1876700024788224/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763989 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763964.279636:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872c380 x1876700024788224/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763989 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763964.279656:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763964.279679:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789763964.279686:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024788224:192.168.203.107@tcp:250:kworker.0 00000100:00100000:2.0:1789763964.279728:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024788352:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789763964.279797:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789763964.279828:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763964/real 1789763964] req@ffffa06c1872ce00 x1876700024788352/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763989 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763964.279922:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872ce00 x1876700024788352/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763989 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763964.279944:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763964.279953:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789763964.279988:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024788352:192.168.203.107@tcp:38:kworker.0 00000100:00100000:2.0:1789763964.280036:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024788480:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789763964.280114:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789763964.280143:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763964/real 1789763964] req@ffffa06c1872e300 x1876700024788480/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763989 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763964.280211:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872e300 x1876700024788480/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763989 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763964.280246:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763964.280253:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789763964.280259:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024788480:192.168.203.107@tcp:38:kworker.0 00010000:00010000:1.0:1789763964.535059:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763964.790960:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789763965.559025:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763965.814995:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789763966.583090:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763966.838969:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789763967.607048:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763967.863005:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789763968.631012:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763968.886970:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:1.0:1789763969.399061:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763969.399072:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789763969.399076:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763969.399087:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 3403 00000100:00080000:1.0:1789763969.399130:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763969.399140:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789763969.399178:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2c000 x1876700024788608/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763969.399207:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763969.399211:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763969.399213:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763969.399217:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 3403 00000100:00080000:1.0:1789763969.399221:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763969.399225:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789763969.399234:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763969.399246:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2e300 x1876700024788736/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763969.399260:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763969.399263:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763969.399265:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1789763969.399272:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024788608:192.168.203.107@tcp:250:kworker.0 00000100:00080000:1.0:1789763969.399273:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 3403 00000100:00080000:1.0:1789763969.399280:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763969.399289:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789763969.399294:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763969.399304:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2ca80 x1876700024788864/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789763969.399381:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789763969.399421:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763969/real 1789763969] req@ffffa06c08e2c000 x1876700024788608/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763994 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763969.399440:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2c000 x1876700024788608/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763994 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763969.399451:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763969.399455:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789763969.399459:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024788608:192.168.203.107@tcp:250:kworker.0 00000100:00100000:2.0:1789763969.399482:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024788736:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789763969.399503:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789763969.399514:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763969/real 1789763969] req@ffffa06c08e2e300 x1876700024788736/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763994 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763969.399526:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2e300 x1876700024788736/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763994 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763969.399533:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763969.399536:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789763969.399549:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024788736:192.168.203.107@tcp:38:kworker.0 00000100:00100000:2.0:1789763969.399555:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024788864:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789763969.399576:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789763969.399585:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763969/real 1789763969] req@ffffa06c08e2ca80 x1876700024788864/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763994 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763969.399595:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2ca80 x1876700024788864/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763994 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763969.399603:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763969.399606:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789763969.399609:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024788864:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789763969.655044:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763969.910988:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789763970.679009:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763970.934952:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789763971.703039:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763971.958977:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789763972.727032:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763972.982919:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789763973.751058:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763974.006961:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789763974.518979:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763974.518990:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763974.518995:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763974.519007:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 3408 00000100:00080000:2.0:1789763974.519020:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763974.519030:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789763974.519092:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e52680 x1876700024788992/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763974.519124:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763974.519128:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763974.519130:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763974.519134:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 3408 00000100:00080000:2.0:1789763974.519139:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763974.519161:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789763974.519172:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763974.519187:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e53800 x1876700024789120/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763974.519192:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e52680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024788992:192.168.203.107@tcp:250:kworker.0 00000100:00080000:2.0:1789763974.519202:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763974.519211:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763974.519215:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763974.519220:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 3408 00000100:00080000:2.0:1789763974.519224:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763974.519227:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789763974.519233:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763974.519245:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51c00 x1876700024789248/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789763974.519278:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789763974.519320:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763974/real 1789763974] req@ffffa06c07e52680 x1876700024788992/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763999 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763974.519337:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e52680 x1876700024788992/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763999 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763974.519347:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763974.519351:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789763974.519355:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e52680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024788992:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789763974.519378:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e53800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024789120:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789763974.519396:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789763974.519405:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763974/real 1789763974] req@ffffa06c07e53800 x1876700024789120/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763999 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763974.519416:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e53800 x1876700024789120/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763999 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763974.519424:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763974.519427:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789763974.519430:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e53800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024789120:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789763974.519435:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024789248:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789763974.519457:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789763974.519471:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763974/real 1789763974] req@ffffa06c07e51c00 x1876700024789248/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763999 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763974.519508:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51c00 x1876700024789248/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763999 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763974.519522:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763974.519526:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789763974.519531:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024789248:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789763974.775035:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763975.030973:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789763975.799021:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763976.054976:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763976.823028:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763977.078932:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763977.846986:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763978.102959:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763978.870941:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763979.126920:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:3.0:1789763979.638959:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763979.638967:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763979.638970:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763979.638978:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 3413 00000100:00080000:3.0:1789763979.638987:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763979.638993:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:3.0:1789763979.639024:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872d180 x1876700024789376/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763979.639083:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763979.639087:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763979.639090:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763979.639095:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 3413 00000100:00080000:3.0:1789763979.639099:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763979.639102:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789763979.639110:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763979.639132:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024789376:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789763979.639132:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872ca80 x1876700024789504/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763979.639149:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763979.639151:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763979.639153:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763979.639156:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 3413 00000100:00080000:3.0:1789763979.639158:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763979.639160:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789763979.639165:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763979.639173:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872f100 x1876700024789632/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789763979.639195:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789763979.639235:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763979/real 1789763979] req@ffffa06c1872d180 x1876700024789376/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764004 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763979.639251:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872d180 x1876700024789376/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764004 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763979.639259:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763979.639262:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789763979.639266:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024789376:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789763979.639284:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024789504:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789763979.639297:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789763979.639305:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763979/real 1789763979] req@ffffa06c1872ca80 x1876700024789504/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764004 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763979.639313:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872ca80 x1876700024789504/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764004 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763979.639329:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763979.639332:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789763979.639334:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024789504:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789763979.639339:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024789632:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789763979.639352:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789763979.639358:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763979/real 1789763979] req@ffffa06c1872f100 x1876700024789632/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764004 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763979.639366:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872f100 x1876700024789632/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764004 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763979.639371:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763979.639372:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789763979.639374:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024789632:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789763979.894977:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763980.150965:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789763980.919087:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763981.174924:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763981.942957:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763982.198987:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763982.966999:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763983.222934:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789763983.991128:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763984.246933:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:1.0:1789763984.759057:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763984.759066:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789763984.759069:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763984.759077:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 3418 00000100:00080000:1.0:1789763984.759086:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763984.759092:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789763984.759139:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2d500 x1876700024789760/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763984.759175:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763984.759192:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763984.759195:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763984.759201:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 3418 00000100:00080000:1.0:1789763984.759206:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763984.759211:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789763984.759221:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763984.759235:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024789760:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789763984.759243:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2ca80 x1876700024789888/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763984.759259:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763984.759264:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763984.759266:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763984.759269:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 3418 00000100:00080000:1.0:1789763984.759274:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763984.759279:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789763984.759286:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763984.759300:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2e300 x1876700024790016/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789763984.759326:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789763984.759375:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763984/real 1789763984] req@ffffa06c08e2d500 x1876700024789760/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764009 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763984.759392:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2d500 x1876700024789760/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764009 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763984.759406:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763984.759426:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789763984.759431:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024789760:192.168.203.107@tcp:250:kworker.0 00000100:00100000:2.0:1789763984.759465:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024789888:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789763984.759494:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789763984.759510:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763984/real 1789763984] req@ffffa06c08e2ca80 x1876700024789888/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764009 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763984.759525:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2ca80 x1876700024789888/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764009 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763984.759535:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763984.759538:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789763984.759541:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024789888:192.168.203.107@tcp:38:kworker.0 00000100:00100000:2.0:1789763984.759546:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024790016:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789763984.759564:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789763984.759573:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763984/real 1789763984] req@ffffa06c08e2e300 x1876700024790016/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764009 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763984.759581:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2e300 x1876700024790016/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764009 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763984.759587:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763984.759589:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789763984.759591:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024790016:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789763985.014969:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763985.270953:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763986.038960:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763986.294920:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789763987.063025:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763987.318992:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763988.087005:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763988.342946:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763989.111031:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763989.366952:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:1.0:1789763989.878911:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763989.878923:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789763989.878928:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763989.878941:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 3423 00000100:00080000:1.0:1789763989.878954:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763989.878976:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789763989.879031:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2c000 x1876700024790144/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763989.879077:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763989.879082:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763989.879084:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763989.879089:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 3423 00000100:00080000:1.0:1789763989.879094:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763989.879099:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789763989.879110:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763989.879124:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2ed80 x1876700024790272/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763989.879140:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763989.879145:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763989.879148:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763989.879153:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 3423 00000100:00080000:1.0:1789763989.879156:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763989.879165:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789763989.879167:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024790144:192.168.203.107@tcp:250:kworker.0 00000008:00100000:1.0:1789763989.879171:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763989.879193:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2c700 x1876700024790400/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789763989.879253:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789763989.879298:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763989/real 1789763989] req@ffffa06c08e2c000 x1876700024790144/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764014 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763989.879318:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2c000 x1876700024790144/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764014 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763989.879331:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763989.879335:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789763989.879340:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024790144:192.168.203.107@tcp:250:kworker.0 00000100:00100000:2.0:1789763989.879348:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024790272:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789763989.879369:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789763989.879380:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763989/real 1789763989] req@ffffa06c08e2ed80 x1876700024790272/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764014 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763989.879402:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2ed80 x1876700024790272/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764014 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763989.879410:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763989.879414:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789763989.879418:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024790272:192.168.203.107@tcp:38:kworker.0 00000100:00100000:2.0:1789763989.879442:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024790400:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789763989.879460:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789763989.879471:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763989/real 1789763989] req@ffffa06c08e2c700 x1876700024790400/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764014 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763989.879503:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2c700 x1876700024790400/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764014 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763989.879511:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763989.879514:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789763989.879518:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024790400:192.168.203.107@tcp:38:kworker.0 00010000:00010000:1.0:1789763990.135033:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763990.390929:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763991.159001:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763991.414920:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763992.183008:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763992.439000:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789763993.206990:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763993.462973:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763994.230964:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763994.486997:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789763994.999021:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763994.999032:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763994.999036:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763994.999047:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 3428 00000100:00080000:2.0:1789763994.999060:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763994.999068:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789763994.999115:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e53480 x1876700024790528/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763994.999148:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763994.999151:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763994.999152:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763994.999155:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 3428 00000100:00080000:2.0:1789763994.999158:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763994.999162:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789763994.999172:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763994.999182:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50a80 x1876700024790656/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763994.999204:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e53480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024790528:192.168.203.107@tcp:250:kworker.0 00000100:00080000:2.0:1789763994.999213:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763994.999217:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763994.999219:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763994.999223:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 3428 00000100:00080000:2.0:1789763994.999226:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763994.999230:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789763994.999234:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763994.999245:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50700 x1876700024790784/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789763994.999285:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789763994.999324:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763994/real 1789763994] req@ffffa06c07e53480 x1876700024790528/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764019 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763994.999350:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e53480 x1876700024790528/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764019 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763994.999359:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763994.999362:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789763994.999365:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e53480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024790528:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789763994.999385:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024790656:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789763994.999401:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789763994.999410:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763994/real 1789763994] req@ffffa06c07e50a80 x1876700024790656/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764019 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763994.999418:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50a80 x1876700024790656/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764019 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763994.999424:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763994.999426:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789763994.999428:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024790656:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789763994.999432:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024790784:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789763994.999442:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789763994.999448:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763994/real 1789763994] req@ffffa06c07e50700 x1876700024790784/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764019 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763994.999455:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50700 x1876700024790784/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764019 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763994.999469:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763994.999471:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789763994.999474:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024790784:192.168.203.107@tcp:38:kworker.0 00010000:00010000:2.0:1789763995.255071:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763995.511060:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789763996.278964:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763996.534934:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763997.303024:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763997.558974:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789763998.326989:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763998.582892:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789763999.350991:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763999.606998:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:3.0:1789764000.119011:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764000.119023:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789764000.119029:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764000.119044:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 3433 00000100:00080000:3.0:1789764000.119060:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764000.119092:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:3.0:1789764000.119136:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872c700 x1876700024790912/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764000.119163:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764000.119166:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789764000.119169:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764000.119174:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 3433 00000100:00080000:3.0:1789764000.119180:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764000.119184:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789764000.119194:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764000.119201:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024790912:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789764000.119212:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872e680 x1876700024791040/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764000.119227:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764000.119230:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789764000.119232:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764000.119237:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 3433 00000100:00080000:3.0:1789764000.119240:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764000.119243:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789764000.119250:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764000.119261:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872dc00 x1876700024791168/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789764000.119271:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789764000.119309:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764000/real 1789764000] req@ffffa06c1872c700 x1876700024790912/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764025 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764000.119327:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872c700 x1876700024790912/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764025 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764000.119339:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764000.119344:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789764000.119351:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024790912:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789764000.119381:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024791040:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789764000.119400:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789764000.119410:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764000/real 1789764000] req@ffffa06c1872e680 x1876700024791040/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764025 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764000.119418:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872e680 x1876700024791040/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764025 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764000.119424:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764000.119427:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789764000.119440:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024791040:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789764000.119445:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024791168:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789764000.119460:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789764000.119467:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764000/real 1789764000] req@ffffa06c1872dc00 x1876700024791168/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764025 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764000.119475:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872dc00 x1876700024791168/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764025 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764000.119483:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764000.119486:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789764000.119489:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024791168:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789764000.375007:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764000.631029:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789764001.398998:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764001.654927:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789764002.422984:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764002.678945:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789764003.447034:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764003.702980:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789764004.470977:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764004.726943:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:1.0:1789764005.238991:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764005.239001:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789764005.239005:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764005.239016:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 3439 00000100:00080000:1.0:1789764005.239027:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764005.239034:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789764005.239078:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2d180 x1876700024791296/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764005.239107:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764005.239110:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789764005.239111:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764005.239115:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 3439 00000100:00080000:1.0:1789764005.239117:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764005.239142:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789764005.239152:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764005.239163:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024791296:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789764005.239168:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2c700 x1876700024791424/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764005.239179:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764005.239181:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789764005.239183:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764005.239186:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 3439 00000100:00080000:1.0:1789764005.239189:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764005.239191:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789764005.239195:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764005.239208:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2ed80 x1876700024791552/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789764005.239239:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789764005.239281:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764005/real 1789764005] req@ffffa06c08e2d180 x1876700024791296/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764030 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764005.239295:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2d180 x1876700024791296/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764030 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764005.239304:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764005.239307:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789764005.239311:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024791296:192.168.203.107@tcp:250:kworker.0 00000100:00100000:2.0:1789764005.239353:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024791424:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789764005.239378:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789764005.239404:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764005/real 1789764005] req@ffffa06c08e2c700 x1876700024791424/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764030 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764005.239412:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2c700 x1876700024791424/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764030 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764005.239419:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764005.239421:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789764005.239423:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024791424:192.168.203.107@tcp:38:kworker.0 00000100:00100000:2.0:1789764005.239427:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024791552:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789764005.239438:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789764005.239444:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764005/real 1789764005] req@ffffa06c08e2ed80 x1876700024791552/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764030 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764005.239467:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2ed80 x1876700024791552/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764030 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764005.239473:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764005.239474:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789764005.239476:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024791552:192.168.203.107@tcp:38:kworker.0 00010000:00010000:2.0:1789764005.494997:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764005.750932:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789764006.519094:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764006.774915:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789764007.542963:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764007.798943:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789764008.566956:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764008.822923:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789764009.590978:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764009.846958:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:1.0:1789764010.358921:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764010.358931:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789764010.358935:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764010.358946:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 3444 00000100:00080000:1.0:1789764010.358956:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764010.358963:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789764010.358999:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2c000 x1876700024791680/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764010.359026:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764010.359030:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789764010.359046:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764010.359051:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 3444 00000100:00080000:1.0:1789764010.359056:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764010.359061:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789764010.359071:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764010.359087:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2e300 x1876700024791808/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764010.359100:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764010.359104:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789764010.359106:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764010.359111:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 3444 00000100:00080000:1.0:1789764010.359114:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764010.359117:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789764010.359122:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764010.359133:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024791680:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789764010.359136:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2ca80 x1876700024791936/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789764010.359249:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789764010.359337:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764010/real 1789764010] req@ffffa06c08e2c000 x1876700024791680/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764035 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764010.359361:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2c000 x1876700024791680/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764035 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764010.359375:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764010.359380:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789764010.359384:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024791680:192.168.203.107@tcp:250:kworker.0 00000100:00100000:2.0:1789764010.359395:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024791808:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789764010.359420:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789764010.359434:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764010/real 1789764010] req@ffffa06c08e2e300 x1876700024791808/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764035 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764010.359458:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2e300 x1876700024791808/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764035 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764010.359468:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764010.359472:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789764010.359476:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024791808:192.168.203.107@tcp:38:kworker.0 00000100:00100000:2.0:1789764010.359509:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024791936:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789764010.359532:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789764010.359544:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764010/real 1789764010] req@ffffa06c08e2ca80 x1876700024791936/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764035 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764010.359557:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2ca80 x1876700024791936/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764035 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764010.359582:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764010.359585:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789764010.359588:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024791936:192.168.203.107@tcp:38:kworker.0 00010000:00010000:2.0:1789764010.614977:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764010.870952:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789764011.638984:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764011.894960:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789764012.663133:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764012.918961:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789764013.687006:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764013.942945:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789764014.711068:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764014.966955:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789764015.478973:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764015.478986:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789764015.478992:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764015.479005:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 3449 00000100:00080000:2.0:1789764015.479018:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764015.479027:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789764015.479075:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e53100 x1876700024792064/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764015.479107:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764015.479112:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789764015.479114:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764015.479119:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 3449 00000100:00080000:2.0:1789764015.479123:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764015.479128:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789764015.479140:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764015.479149:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e53100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024792064:192.168.203.107@tcp:250:kworker.0 00000100:00100000:2.0:1789764015.479156:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e52a00 x1876700024792192/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764015.479207:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764015.479211:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789764015.479213:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764015.479217:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 3449 00000100:00080000:2.0:1789764015.479222:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764015.479227:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789764015.479233:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764015.479246:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e52d80 x1876700024792320/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789764015.479299:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e52a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024792192:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789764015.479334:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e52d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024792320:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789764015.734963:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764015.990917:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789764016.759026:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764017.014987:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789764017.783079:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764018.038912:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000400:00000100:2.0:1789764018.551092:0:2330:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.107:988 00000400:00000100:2.0:1789764018.551152:0:2330:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.107@tcp at host 192.168.203.107 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1789764018.551174:0:2330:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.107@tcp 00000800:00000100:2.0:1789764018.551246:0:2330:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.107@tcp 00000800:00000100:2.0:1789764018.551259:0:2330:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.107@tcp 00000100:00100000:1.0:1789764018.551345:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764015/real 1789764018] req@ffffa06c07e52a00 x1876700024792192/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764040 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764018.551401:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e52a00 x1876700024792192/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764040 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764018.551416:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764018.551421:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789764018.551427:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e52a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024792192:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789764018.551436:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764015/real 1789764018] req@ffffa06c07e52d80 x1876700024792320/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764040 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764018.551451:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e52d80 x1876700024792320/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764040 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764018.551459:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764018.551464:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789764018.551467:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e52d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024792320:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789764018.551475:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764015/real 1789764018] req@ffffa06c07e53100 x1876700024792064/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764040 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764018.551486:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e53100 x1876700024792064/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764040 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764018.551494:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764018.551497:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789764018.551501:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e53100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024792064:192.168.203.107@tcp:250:kworker.0 00010000:00010000:3.0:1789764018.806960:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789764019.062983:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789764019.831007:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789764020.087010:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:1.0:1789764020.599031:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764020.599052:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789764020.599056:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764020.599065:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 3454 00000100:00080000:1.0:1789764020.599084:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764020.599095:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789764020.599125:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2d500 x1876700024792448/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764020.599161:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764020.599165:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789764020.599167:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764020.599171:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 3454 00000100:00080000:1.0:1789764020.599174:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764020.599178:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789764020.599187:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764020.599202:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2ca80 x1876700024792576/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764020.599214:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1789764020.599219:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024792448:192.168.203.107@tcp:250:kworker.0 00000100:00080000:1.0:1789764020.599220:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789764020.599224:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764020.599231:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 3454 00000100:00080000:1.0:1789764020.599234:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764020.599238:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789764020.599242:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764020.599253:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2e300 x1876700024792704/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789764020.599288:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789764020.599333:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764020/real 1789764020] req@ffffa06c08e2d500 x1876700024792448/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764045 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764020.599346:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2d500 x1876700024792448/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764045 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764020.599353:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764020.599363:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789764020.599371:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024792448:192.168.203.107@tcp:250:kworker.0 00000100:00100000:2.0:1789764020.599399:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024792576:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789764020.599422:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789764020.599433:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764020/real 1789764020] req@ffffa06c08e2ca80 x1876700024792576/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764045 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764020.599445:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2ca80 x1876700024792576/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764045 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764020.599452:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764020.599455:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789764020.599458:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024792576:192.168.203.107@tcp:38:kworker.0 00000100:00100000:2.0:1789764020.599464:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024792704:192.168.203.107@tcp:38:kworker.0 00000800:00000100:2.0:1789764020.599481:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:2.0:1789764020.599491:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764020/real 1789764020] req@ffffa06c08e2e300 x1876700024792704/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764045 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764020.599501:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2e300 x1876700024792704/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764045 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764020.599509:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764020.599512:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:2.0:1789764020.599515:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024792704:192.168.203.107@tcp:38:kworker.0 00010000:00010000:1.0:1789764020.855012:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789764021.110940:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789764021.879008:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789764022.135096:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789764022.903046:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789764023.159002:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789764023.927007:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789764024.182924:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789764024.951054:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789764025.206916:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789764025.718970:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764025.718983:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789764025.718986:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764025.718994:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 3459 00000100:00080000:2.0:1789764025.719001:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764025.719006:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789764025.719035:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e53100 x1876700024792832/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764025.719055:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764025.719057:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789764025.719058:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764025.719061:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 3459 00000100:00080000:2.0:1789764025.719063:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764025.719065:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789764025.719072:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764025.719079:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e52d80 x1876700024792960/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764025.719086:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764025.719089:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789764025.719091:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1789764025.719094:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e53100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024792832:192.168.203.107@tcp:250:kworker.0 00000100:00080000:2.0:1789764025.719095:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 3459 00000100:00080000:2.0:1789764025.719098:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764025.719102:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789764025.719116:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764025.719124:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e52a00 x1876700024793088/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789764025.719154:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789764025.719187:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764025/real 1789764025] req@ffffa06c07e53100 x1876700024792832/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764050 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764025.719203:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e53100 x1876700024792832/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764050 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764025.719212:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764025.719216:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789764025.719219:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e53100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024792832:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789764025.719238:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e52d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024792960:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789764025.719254:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789764025.719264:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764025/real 1789764025] req@ffffa06c07e52d80 x1876700024792960/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764050 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764025.719274:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e52d80 x1876700024792960/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764050 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764025.719282:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764025.719285:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789764025.719288:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e52d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024792960:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789764025.719293:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e52a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024793088:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789764025.719315:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789764025.719323:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764025/real 1789764025] req@ffffa06c07e52a00 x1876700024793088/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764050 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764025.719334:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e52a00 x1876700024793088/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764050 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764025.719342:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764025.719344:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789764025.719347:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e52a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024793088:192.168.203.107@tcp:38:kworker.0 00010000:00010000:2.0:1789764025.975782:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789764026.230941:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789764026.999066:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764027.254975:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789764028.023156:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764028.278951:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789764029.046952:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764029.302913:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789764030.070952:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764030.326942:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789764030.838952:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764030.838967:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789764030.838971:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764030.838980:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 3464 00000100:00080000:0.0:1789764030.838990:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764030.838997:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789764030.839029:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d3100 x1876700024793216/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764030.839049:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764030.839052:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789764030.839053:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764030.839056:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 3464 00000100:00080000:0.0:1789764030.839058:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764030.839061:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789764030.839067:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764030.839088:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0380 x1876700024793344/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764030.839097:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764030.839099:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:1.0:1789764030.839099:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024793216:192.168.203.107@tcp:250:kworker.0 00000100:00080000:0.0:1789764030.839101:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764030.839108:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 3464 00000100:00080000:0.0:1789764030.839111:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764030.839113:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789764030.839116:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764030.839123:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d2680 x1876700024793472/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789764030.839163:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789764030.839201:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764030/real 1789764030] req@ffffa06c091d3100 x1876700024793216/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764055 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764030.839218:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d3100 x1876700024793216/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764055 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764030.839229:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764030.839234:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789764030.839237:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024793216:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789764030.839258:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024793344:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789764030.839280:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789764030.839292:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764030/real 1789764030] req@ffffa06c091d0380 x1876700024793344/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764055 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764030.839319:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0380 x1876700024793344/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764055 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764030.839327:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764030.839330:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789764030.839333:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024793344:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789764030.839339:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024793472:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789764030.839357:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789764030.839368:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764030/real 1789764030] req@ffffa06c091d2680 x1876700024793472/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764055 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764030.839379:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d2680 x1876700024793472/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764055 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764030.839386:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764030.839388:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789764030.839391:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024793472:192.168.203.107@tcp:38:kworker.0 00010000:00010000:1.0:1789764031.094947:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764031.350941:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789764032.118958:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764032.374912:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789764033.142943:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764033.398924:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789764034.166935:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764034.422932:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789764035.190948:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764035.446949:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789764035.958956:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764035.958963:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789764035.958967:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764035.958975:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 3469 00000100:00080000:2.0:1789764035.958983:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764035.958990:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789764035.959022:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50700 x1876700024793600/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764035.959044:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764035.959047:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789764035.959049:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764035.959052:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 3469 00000100:00080000:2.0:1789764035.959054:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764035.959056:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789764035.959062:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764035.959072:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50a80 x1876700024793728/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789764035.959087:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024793600:192.168.203.107@tcp:250:kworker.0 00000100:00080000:2.0:1789764035.959104:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764035.959106:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789764035.959108:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764035.959111:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 3469 00000100:00080000:2.0:1789764035.959114:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764035.959118:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789764035.959121:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764035.959130:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e53480 x1876700024793856/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789764035.959153:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789764035.959492:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764035/real 1789764035] req@ffffa06c07e50700 x1876700024793600/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764060 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764035.959510:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50700 x1876700024793600/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764060 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764035.959521:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764035.959525:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789764035.959529:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024793600:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789764035.959597:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024793728:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789764035.959618:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789764035.959643:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764035/real 1789764035] req@ffffa06c07e50a80 x1876700024793728/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764060 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764035.959655:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50a80 x1876700024793728/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764060 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764035.959663:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764035.959666:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789764035.959669:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024793728:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789764035.959675:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e53480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024793856:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789764035.959701:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789764035.959710:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764035/real 1789764035] req@ffffa06c07e53480 x1876700024793856/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764060 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764035.959722:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e53480 x1876700024793856/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764060 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764035.959729:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764035.959732:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789764035.959735:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e53480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024793856:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789764036.214995:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764036.470949:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789764037.238994:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764037.494932:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789764038.263069:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764038.518979:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789764039.286997:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764039.542971:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789764040.310961:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764040.566971:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:1.0:1789764041.078951:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764041.078960:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789764041.078964:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764041.078973:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 3474 00000100:00080000:1.0:1789764041.078982:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764041.078989:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789764041.079033:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2c000 x1876700024793984/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764041.079059:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764041.079063:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789764041.079065:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764041.079069:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 3474 00000100:00080000:1.0:1789764041.079073:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764041.079077:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789764041.079085:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764041.079098:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024793984:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789764041.079100:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2ed80 x1876700024794112/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764041.079113:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764041.079117:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789764041.079118:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764041.079122:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 3474 00000100:00080000:1.0:1789764041.079125:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764041.079129:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789764041.079134:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764041.079145:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2c700 x1876700024794240/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789764041.079148:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789764041.079188:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764041/real 1789764041] req@ffffa06c08e2c000 x1876700024793984/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764066 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764041.079200:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2c000 x1876700024793984/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764066 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764041.079206:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764041.079214:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789764041.079216:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024793984:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789764041.079233:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024794112:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789764041.079248:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789764041.079255:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764041/real 1789764041] req@ffffa06c08e2ed80 x1876700024794112/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764066 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764041.079262:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2ed80 x1876700024794112/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764066 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764041.079268:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764041.079269:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789764041.079271:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024794112:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789764041.079274:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024794240:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789764041.079283:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789764041.079289:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764041/real 1789764041] req@ffffa06c08e2c700 x1876700024794240/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764066 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764041.079295:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2c700 x1876700024794240/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764066 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764041.079299:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764041.079300:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789764041.079302:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024794240:192.168.203.107@tcp:38:kworker.0 00010000:00010000:2.0:1789764041.335064:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764041.590951:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789764042.358968:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764042.614933:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789764043.382974:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764043.638987:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789764044.406946:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764044.663035:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789764045.430994:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764045.686912:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789764046.198943:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764046.198951:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789764046.198954:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764046.198962:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 3480 00000100:00080000:2.0:1789764046.198971:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764046.198978:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789764046.199012:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51c00 x1876700024794368/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764046.199032:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764046.199035:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789764046.199036:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764046.199050:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 3480 00000100:00080000:2.0:1789764046.199054:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764046.199057:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789764046.199064:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764046.199068:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024794368:192.168.203.107@tcp:250:kworker.0 00000100:00100000:2.0:1789764046.199076:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e53800 x1876700024794496/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764046.199085:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764046.199087:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789764046.199088:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764046.199090:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 3480 00000100:00080000:2.0:1789764046.199093:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764046.199095:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789764046.199098:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764046.199105:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e52680 x1876700024794624/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789764046.199120:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789764046.199158:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764046/real 1789764046] req@ffffa06c07e51c00 x1876700024794368/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764071 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764046.199170:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51c00 x1876700024794368/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764071 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764046.199177:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764046.199179:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789764046.199182:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024794368:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789764046.199198:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e53800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024794496:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789764046.199211:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789764046.199219:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764046/real 1789764046] req@ffffa06c07e53800 x1876700024794496/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764071 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764046.199226:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e53800 x1876700024794496/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764071 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764046.199231:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764046.199233:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789764046.199235:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e53800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024794496:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789764046.199244:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e52680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024794624:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789764046.199256:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789764046.199262:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764046/real 1789764046] req@ffffa06c07e52680 x1876700024794624/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764071 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764046.199274:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e52680 x1876700024794624/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764071 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764046.199279:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764046.199281:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789764046.199283:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e52680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024794624:192.168.203.107@tcp:38:kworker.0 00010000:00010000:1.0:1789764046.454953:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764046.710944:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789764047.478968:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764047.734895:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789764048.503016:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764048.759117:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789764049.526952:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764049.782952:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789764050.550973:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764050.807002:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789764051.319032:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764051.319048:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789764051.319054:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764051.319068:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 3485 00000100:00080000:2.0:1789764051.319083:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764051.319093:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789764051.319156:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50380 x1876700024794752/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764051.319185:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764051.319206:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789764051.319208:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764051.319214:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 3485 00000100:00080000:2.0:1789764051.319219:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764051.319226:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:3.0:1789764051.319233:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024794752:192.168.203.107@tcp:250:kworker.0 00000008:00100000:2.0:1789764051.319241:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764051.319265:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e53b80 x1876700024794880/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764051.319281:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764051.319285:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789764051.319287:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764051.319293:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 3485 00000100:00080000:2.0:1789764051.319298:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1789764051.319300:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00080000:2.0:1789764051.319309:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789764051.319315:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764051.319327:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51f80 x1876700024795008/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789764051.319351:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764051/real 1789764051] req@ffffa06c07e50380 x1876700024794752/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764076 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764051.319369:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50380 x1876700024794752/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764076 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764051.319376:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764051.319379:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789764051.319389:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024794752:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789764051.319415:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e53b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024794880:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789764051.319434:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789764051.319442:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764051/real 1789764051] req@ffffa06c07e53b80 x1876700024794880/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764076 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764051.319450:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e53b80 x1876700024794880/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764076 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764051.319455:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764051.319457:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789764051.319459:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e53b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024794880:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789764051.319463:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024795008:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789764051.319486:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789764051.319497:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764051/real 1789764051] req@ffffa06c07e51f80 x1876700024795008/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764076 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764051.319509:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51f80 x1876700024795008/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764076 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764051.319541:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764051.319544:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789764051.319546:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024795008:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789764051.574978:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764051.830953:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789764052.599046:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764052.854970:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789764053.623005:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764053.878991:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789764054.646959:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764054.902958:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789764055.671009:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764055.926958:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789764056.438951:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764056.438964:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789764056.438969:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764056.438987:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 3490 00000100:00080000:0.0:1789764056.439001:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764056.439009:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789764056.439054:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d2a00 x1876700024795136/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764056.439077:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764056.439080:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789764056.439097:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764056.439101:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 3490 00000100:00080000:0.0:1789764056.439104:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764056.439108:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789764056.439119:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764056.439140:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0e00 x1876700024795264/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789764056.439150:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024795136:192.168.203.107@tcp:250:kworker.0 00000100:00080000:0.0:1789764056.439160:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764056.439164:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789764056.439166:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764056.439170:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 3490 00000100:00080000:0.0:1789764056.439174:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764056.439178:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789764056.439184:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764056.439200:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d1500 x1876700024795392/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789764056.439219:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789764056.439275:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764056/real 1789764056] req@ffffa06c091d2a00 x1876700024795136/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764081 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764056.439293:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d2a00 x1876700024795136/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764081 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764056.439302:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764056.439305:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789764056.439307:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024795136:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789764056.439324:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024795264:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789764056.439342:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789764056.439359:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764056/real 1789764056] req@ffffa06c091d0e00 x1876700024795264/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764081 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764056.439369:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0e00 x1876700024795264/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764081 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764056.439377:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764056.439381:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789764056.439384:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024795264:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789764056.439390:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024795392:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789764056.439419:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789764056.439427:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764056/real 1789764056] req@ffffa06c091d1500 x1876700024795392/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764081 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764056.439433:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d1500 x1876700024795392/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764081 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764056.439438:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764056.439440:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789764056.439442:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024795392:192.168.203.107@tcp:38:kworker.0 00010000:00010000:0.0:1789764056.695076:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764056.950936:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789764057.718977:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764057.974977:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789764058.743747:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764058.998925:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789764059.767837:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764060.022960:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789764060.791018:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764061.046900:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789764061.558962:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764061.558970:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789764061.558974:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764061.558982:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 3495 00000100:00080000:2.0:1789764061.558993:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764061.558998:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789764061.559042:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50000 x1876700024795520/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764061.559067:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764061.559071:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789764061.559072:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764061.559076:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 3495 00000100:00080000:2.0:1789764061.559079:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764061.559083:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789764061.559091:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764061.559103:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e50e00 x1876700024795648/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789764061.559106:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024795520:192.168.203.107@tcp:250:kworker.0 00000100:00080000:2.0:1789764061.559120:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764061.559124:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789764061.559126:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764061.559130:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 3495 00000100:00080000:2.0:1789764061.559134:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764061.559137:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789764061.559141:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764061.559151:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51500 x1876700024795776/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789764061.559161:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789764061.559194:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764061/real 1789764061] req@ffffa06c07e50000 x1876700024795520/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764086 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764061.559205:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50000 x1876700024795520/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764086 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764061.559211:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764061.559214:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789764061.559216:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024795520:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789764061.559238:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e50e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024795648:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789764061.559252:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789764061.559259:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764061/real 1789764061] req@ffffa06c07e50e00 x1876700024795648/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764086 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764061.559266:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e50e00 x1876700024795648/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764086 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764061.559271:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764061.559273:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789764061.559275:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e50e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024795648:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789764061.559278:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024795776:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789764061.559287:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789764061.559292:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764061/real 1789764061] req@ffffa06c07e51500 x1876700024795776/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764086 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764061.559297:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51500 x1876700024795776/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764086 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764061.559302:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764061.559303:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789764061.559305:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024795776:192.168.203.107@tcp:38:kworker.0 00010000:00010000:2.0:1789764061.815029:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764062.070943:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789764062.839483:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764063.094944:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789764063.862960:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764064.118988:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789764064.887093:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764065.143568:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789764065.911482:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764066.167031:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:1.0:1789764066.679073:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764066.679080:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789764066.679083:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764066.679091:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 3500 00000100:00080000:1.0:1789764066.679099:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764066.679104:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789764066.679147:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2d180 x1876700024795904/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764066.679184:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764066.679186:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789764066.679188:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764066.679191:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 3500 00000100:00080000:1.0:1789764066.679194:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764066.679197:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789764066.679204:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764066.679215:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2c700 x1876700024796032/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764066.679222:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764066.679224:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789764066.679225:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764066.679227:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 3500 00000100:00080000:1.0:1789764066.679228:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764066.679230:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789764066.679245:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764066.679251:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2ed80 x1876700024796160/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789764066.679363:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024795904:192.168.203.107@tcp:250:kworker.0 00000800:00000100:3.0:1789764066.679417:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789764066.679458:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764066/real 1789764066] req@ffffa06c08e2d180 x1876700024795904/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764091 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764066.679484:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2d180 x1876700024795904/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764091 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764066.679494:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764066.679498:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789764066.679502:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024795904:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789764066.679509:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024796032:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789764066.679532:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789764066.679544:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764066/real 1789764066] req@ffffa06c08e2c700 x1876700024796032/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764091 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764066.679556:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2c700 x1876700024796032/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764091 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764066.679564:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764066.679567:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789764066.679570:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024796032:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789764066.679576:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024796160:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789764066.679599:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789764066.679628:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764066/real 1789764066] req@ffffa06c08e2ed80 x1876700024796160/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764091 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764066.679639:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2ed80 x1876700024796160/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764091 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764066.679644:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764066.679648:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789764066.679651:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024796160:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789764066.934971:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764067.190996:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789764067.959050:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764068.215510:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789764068.984085:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764069.239018:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789764070.006995:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764070.262958:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789764071.030993:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764071.286955:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789764071.799295:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764071.799302:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789764071.799306:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764071.799315:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 3505 00000100:00080000:2.0:1789764071.799324:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764071.799329:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789764071.799353:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51180 x1876700024796288/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764071.799517:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764071.799520:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789764071.799522:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764071.799526:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 3505 00000100:00080000:2.0:1789764071.799530:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764071.799534:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789764071.799542:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764071.799554:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e52300 x1876700024796416/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764071.799563:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764071.799566:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789764071.799567:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764071.799570:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 3505 00000100:00080000:2.0:1789764071.799572:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764071.799575:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789764071.799578:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764071.799596:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51880 x1876700024796544/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789764071.799873:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024796288:192.168.203.107@tcp:250:kworker.0 00000800:00000100:1.0:1789764071.799920:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789764071.799944:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764071/real 1789764071] req@ffffa06c07e51180 x1876700024796288/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764096 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764071.799955:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51180 x1876700024796288/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764096 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764071.799961:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764071.799964:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789764071.799966:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024796288:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789764071.799971:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e52300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024796416:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789764071.799983:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789764071.799989:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764071/real 1789764071] req@ffffa06c07e52300 x1876700024796416/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764096 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764071.799996:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e52300 x1876700024796416/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764096 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764071.800000:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764071.800002:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789764071.800004:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e52300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024796416:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789764071.800007:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024796544:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789764071.800031:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789764071.800041:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764071/real 1789764071] req@ffffa06c07e51880 x1876700024796544/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764096 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764071.800052:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51880 x1876700024796544/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764096 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764071.800059:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764071.800062:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789764071.800065:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024796544:192.168.203.107@tcp:38:kworker.0 00010000:00010000:0.0:1789764072.055095:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764072.310942:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789764073.078961:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764073.334948:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789764074.102952:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764074.359069:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789764075.127438:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764075.382947:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789764076.151018:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764076.406920:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:1.0:1789764076.919193:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764076.919203:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789764076.919207:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764076.919222:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 3510 00000100:00080000:1.0:1789764076.919232:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764076.919240:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789764076.919278:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2c000 x1876700024796672/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764076.919311:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764076.919315:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789764076.919317:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764076.919320:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 3510 00000100:00080000:1.0:1789764076.919324:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764076.919346:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789764076.919355:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764076.919380:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2e300 x1876700024796800/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764076.919391:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764076.919394:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789764076.919396:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764076.919400:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 3510 00000100:00080000:1.0:1789764076.919404:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764076.919409:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789764076.919414:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764076.919426:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2ca80 x1876700024796928/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789764076.919429:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024796672:192.168.203.107@tcp:250:kworker.0 00000800:00000100:3.0:1789764076.919488:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789764076.919522:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764076/real 1789764076] req@ffffa06c08e2c000 x1876700024796672/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764101 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764076.919534:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2c000 x1876700024796672/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764101 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764076.919546:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764076.919548:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789764076.919551:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024796672:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789764076.919556:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024796800:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789764076.919575:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789764076.919587:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764076/real 1789764076] req@ffffa06c08e2e300 x1876700024796800/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764101 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764076.919599:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2e300 x1876700024796800/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764101 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764076.919607:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764076.919611:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789764076.919614:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024796800:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789764076.919631:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024796928:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789764076.919644:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789764076.919651:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764076/real 1789764076] req@ffffa06c08e2ca80 x1876700024796928/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764101 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764076.919658:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2ca80 x1876700024796928/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764101 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764076.919665:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764076.919668:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789764076.919671:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024796928:192.168.203.107@tcp:38:kworker.0 00010000:00010000:2.0:1789764077.174975:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764077.430960:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789764078.198962:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764078.455216:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789764079.223822:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764079.479226:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789764080.247526:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764080.502924:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789764081.271460:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764081.526956:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:3.0:1789764082.038976:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764082.038984:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789764082.038988:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764082.038997:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 3515 00000100:00080000:3.0:1789764082.039007:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764082.039013:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:3.0:1789764082.039048:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872ea00 x1876700024797056/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764082.039073:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764082.039077:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789764082.039078:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764082.039087:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 3515 00000100:00080000:3.0:1789764082.039092:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764082.039096:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789764082.039106:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764082.039110:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024797056:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789764082.039121:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872f480 x1876700024797184/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764082.039133:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764082.039137:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789764082.039152:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764082.039156:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 3515 00000100:00080000:3.0:1789764082.039161:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764082.039165:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789764082.039169:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764082.039181:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872ed80 x1876700024797312/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789764082.039597:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024797184:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789764082.039624:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024797312:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789764082.295061:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764082.550977:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789764083.319318:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764083.574945:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789764084.343080:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764084.598921:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000400:00000100:1.0:1789764085.111384:0:2331:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.107:988 00000400:00000100:1.0:1789764085.111437:0:2331:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.107@tcp at host 192.168.203.107 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1789764085.111467:0:2331:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.107@tcp 00000800:00000100:1.0:1789764085.111511:0:2331:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.107@tcp 00000800:00000100:1.0:1789764085.111536:0:2331:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.107@tcp 00000100:00100000:3.0:1789764085.112195:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764082/real 1789764085] req@ffffa06c1872f480 x1876700024797184/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764107 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764085.112220:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872f480 x1876700024797184/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764107 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764085.112232:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764085.112237:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789764085.112242:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024797184:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789764085.112255:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764082/real 1789764085] req@ffffa06c1872ed80 x1876700024797312/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764107 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764085.112275:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872ed80 x1876700024797312/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764107 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764085.112283:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764085.112295:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789764085.112298:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024797312:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789764085.112305:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764082/real 1789764085] req@ffffa06c1872ea00 x1876700024797056/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764107 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764085.112319:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872ea00 x1876700024797056/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764107 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764085.112331:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764085.112334:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789764085.112337:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024797056:192.168.203.107@tcp:250:kworker.0 00010000:00010000:3.0:1789764085.366991:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764085.622938:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789764086.391046:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764086.647144:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:3.0:1789764087.158991:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764087.158998:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789764087.159002:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764087.159015:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 3520 00000100:00080000:3.0:1789764087.159023:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764087.159028:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:3.0:1789764087.159053:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872ea00 x1876700024797440/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764087.159235:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764087.159238:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789764087.159240:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764087.159244:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 3520 00000100:00080000:3.0:1789764087.159269:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764087.159274:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789764087.159292:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764087.159295:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024797440:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789764087.159309:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872ed80 x1876700024797568/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764087.159322:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764087.159326:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789764087.159328:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764087.159332:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 3520 00000100:00080000:3.0:1789764087.159336:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764087.159339:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789764087.159344:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1789764087.159357:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789764087.159357:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872f480 x1876700024797696/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789764087.159399:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764087/real 1789764087] req@ffffa06c1872ea00 x1876700024797440/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764112 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764087.159417:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872ea00 x1876700024797440/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764112 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764087.159427:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764087.159431:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789764087.159434:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024797440:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789764087.159688:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024797568:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789764087.159714:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789764087.159727:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764087/real 1789764087] req@ffffa06c1872ed80 x1876700024797568/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764112 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764087.159740:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872ed80 x1876700024797568/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764112 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764087.159748:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764087.159752:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789764087.159755:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024797568:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789764087.159761:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024797696:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789764087.159779:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789764087.159799:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764087/real 1789764087] req@ffffa06c1872f480 x1876700024797696/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764112 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764087.159810:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872f480 x1876700024797696/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764112 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764087.159817:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764087.159819:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789764087.159822:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024797696:192.168.203.107@tcp:38:kworker.0 00010000:00010000:1.0:1789764087.416301:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764087.670970:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789764088.439178:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764088.695144:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789764089.463398:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764089.718953:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789764090.486970:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764090.742902:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789764091.510922:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764091.766958:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:3.0:1789764092.278989:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764092.279001:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789764092.279005:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764092.279017:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 3526 00000100:00080000:3.0:1789764092.279053:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764092.279061:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:3.0:1789764092.279102:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872fb80 x1876700024797824/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764092.279140:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764092.279144:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789764092.279146:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764092.279150:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 3526 00000100:00080000:3.0:1789764092.279154:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764092.279157:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789764092.279168:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764092.279182:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872f480 x1876700024797952/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789764092.279189:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024797824:192.168.203.107@tcp:250:kworker.0 00000100:00080000:3.0:1789764092.279200:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764092.279205:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789764092.279206:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764092.279210:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 3526 00000100:00080000:3.0:1789764092.279227:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764092.279232:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789764092.279237:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764092.279249:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872ed80 x1876700024798080/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789764092.279293:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789764092.279333:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764092/real 1789764092] req@ffffa06c1872fb80 x1876700024797824/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764117 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764092.279350:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872fb80 x1876700024797824/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764117 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764092.279361:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764092.279366:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789764092.279370:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024797824:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789764092.279400:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024797952:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789764092.279417:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789764092.279426:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764092/real 1789764092] req@ffffa06c1872f480 x1876700024797952/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764117 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764092.279444:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872f480 x1876700024797952/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764117 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764092.279450:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764092.279452:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789764092.279454:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024797952:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789764092.279457:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024798080:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789764092.279469:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789764092.279475:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764092/real 1789764092] req@ffffa06c1872ed80 x1876700024798080/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764117 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764092.279482:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872ed80 x1876700024798080/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764117 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764092.279486:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764092.279488:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789764092.279489:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024798080:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789764092.535028:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764092.790955:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789764093.558974:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764093.814954:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789764094.582964:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764094.838946:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789764095.606987:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764095.862948:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789764096.631058:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764096.886953:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:1.0:1789764097.398981:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764097.398988:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789764097.398991:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764097.398998:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 3531 00000100:00080000:1.0:1789764097.399007:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764097.399012:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789764097.399040:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2d500 x1876700024798208/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764097.399073:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764097.399075:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789764097.399076:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764097.399079:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 3531 00000100:00080000:1.0:1789764097.399087:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764097.399090:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789764097.399096:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764097.399108:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2ca80 x1876700024798336/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764097.399116:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764097.399122:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789764097.399123:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764097.399125:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 3531 00000100:00080000:1.0:1789764097.399127:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764097.399131:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789764097.399137:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764097.399139:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024798208:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789764097.399144:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2e300 x1876700024798464/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789764097.399270:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789764097.399325:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764097/real 1789764097] req@ffffa06c08e2d500 x1876700024798208/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764122 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764097.399345:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2d500 x1876700024798208/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764122 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764097.399355:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764097.399360:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789764097.399364:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024798208:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789764097.399372:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024798336:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789764097.399394:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789764097.399406:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764097/real 1789764097] req@ffffa06c08e2ca80 x1876700024798336/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764122 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764097.399417:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2ca80 x1876700024798336/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764122 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764097.399424:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764097.399427:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789764097.399441:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024798336:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789764097.399468:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024798464:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789764097.399488:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789764097.399498:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764097/real 1789764097] req@ffffa06c08e2e300 x1876700024798464/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764122 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764097.399509:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2e300 x1876700024798464/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764122 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764097.399516:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764097.399518:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789764097.399521:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024798464:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789764097.655011:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764097.910912:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789764098.678999:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764098.934970:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789764099.703122:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764099.959011:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789764100.727025:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764100.983010:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789764101.750985:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764102.006931:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:1.0:1789764102.518977:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764102.518987:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789764102.518991:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764102.519003:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 3536 00000100:00080000:1.0:1789764102.519034:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764102.519041:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789764102.519084:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2c000 x1876700024798592/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764102.519108:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764102.519111:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789764102.519112:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764102.519116:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 3536 00000100:00080000:1.0:1789764102.519118:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764102.519120:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789764102.519133:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764102.519156:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2ed80 x1876700024798720/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764102.519166:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764102.519169:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789764102.519170:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764102.519173:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 3536 00000100:00080000:1.0:1789764102.519177:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764102.519185:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:3.0:1789764102.519186:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024798592:192.168.203.107@tcp:250:kworker.0 00000008:00100000:1.0:1789764102.519190:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764102.519207:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2c700 x1876700024798848/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789764102.519268:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789764102.519312:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764102/real 1789764102] req@ffffa06c08e2c000 x1876700024798592/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764127 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764102.519330:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2c000 x1876700024798592/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764127 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764102.519352:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764102.519355:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789764102.519358:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024798592:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789764102.519364:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024798720:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789764102.519380:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789764102.519388:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764102/real 1789764102] req@ffffa06c08e2ed80 x1876700024798720/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764127 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764102.519395:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2ed80 x1876700024798720/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764127 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764102.519400:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764102.519403:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789764102.519405:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024798720:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789764102.519422:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024798848:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789764102.519432:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789764102.519438:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764102/real 1789764102] req@ffffa06c08e2c700 x1876700024798848/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764127 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764102.519444:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2c700 x1876700024798848/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764127 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764102.519449:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764102.519450:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789764102.519452:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024798848:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789764102.774987:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764103.030957:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789764103.799034:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764104.054972:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789764104.822992:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764105.078992:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789764105.847043:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764106.102995:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789764106.871191:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764107.126964:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:1.0:1789764107.638949:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764107.638970:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789764107.638974:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764107.638983:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 3541 00000100:00080000:1.0:1789764107.638993:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764107.639000:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789764107.639029:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2d180 x1876700024798976/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764107.639057:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764107.639060:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789764107.639062:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764107.639066:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 3541 00000100:00080000:1.0:1789764107.639069:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764107.639073:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789764107.639082:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764107.639119:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024798976:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789764107.639119:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2c700 x1876700024799104/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764107.639136:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764107.639140:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789764107.639141:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764107.639145:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 3541 00000100:00080000:1.0:1789764107.639148:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764107.639152:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789764107.639157:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764107.639176:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2ed80 x1876700024799232/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789764107.639185:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789764107.639226:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764107/real 1789764107] req@ffffa06c08e2d180 x1876700024798976/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764132 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764107.639246:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2d180 x1876700024798976/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764132 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764107.639256:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764107.639260:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789764107.639265:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024798976:192.168.203.107@tcp:250:kworker.0 00000100:00100000:0.0:1789764107.639287:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024799104:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789764107.639309:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789764107.639321:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764107/real 1789764107] req@ffffa06c08e2c700 x1876700024799104/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764132 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764107.639334:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2c700 x1876700024799104/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764132 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764107.639343:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764107.639346:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789764107.639349:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024799104:192.168.203.107@tcp:38:kworker.0 00000100:00100000:0.0:1789764107.639355:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024799232:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789764107.639376:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789764107.639388:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764107/real 1789764107] req@ffffa06c08e2ed80 x1876700024799232/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764132 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764107.639412:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2ed80 x1876700024799232/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764132 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764107.639421:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764107.639423:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789764107.639426:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024799232:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789764107.895009:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764108.150965:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789764108.919043:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764109.174942:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789764109.942992:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764110.198975:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789764110.966965:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764111.222952:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789764111.990972:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764112.246943:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:1.0:1789764112.758939:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764112.758953:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789764112.758957:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764112.758966:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 3546 00000100:00080000:1.0:1789764112.758976:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764112.758984:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789764112.759016:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2c000 x1876700024799360/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764112.759039:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764112.759043:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789764112.759044:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764112.759048:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 3546 00000100:00080000:1.0:1789764112.759052:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764112.759063:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789764112.759084:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764112.759086:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024799360:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789764112.759101:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2e300 x1876700024799488/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764112.759113:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764112.759117:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789764112.759119:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764112.759122:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 3546 00000100:00080000:1.0:1789764112.759128:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764112.759132:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000800:00000100:0.0:1789764112.759135:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000008:00100000:1.0:1789764112.759150:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764112.759164:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2ca80 x1876700024799616/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789764112.759175:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764112/real 1789764112] req@ffffa06c08e2c000 x1876700024799360/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764137 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764112.759194:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2c000 x1876700024799360/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764137 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764112.759205:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764112.759209:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789764112.759213:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024799360:192.168.203.107@tcp:250:kworker.0 00000100:00100000:0.0:1789764112.759236:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024799488:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789764112.759258:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789764112.759271:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764112/real 1789764112] req@ffffa06c08e2e300 x1876700024799488/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764137 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764112.759283:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2e300 x1876700024799488/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764137 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764112.759303:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764112.759307:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789764112.759310:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024799488:192.168.203.107@tcp:38:kworker.0 00000100:00100000:0.0:1789764112.759317:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024799616:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789764112.759338:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789764112.759349:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764112/real 1789764112] req@ffffa06c08e2ca80 x1876700024799616/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764137 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764112.759360:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2ca80 x1876700024799616/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764137 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764112.759368:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764112.759371:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789764112.759375:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024799616:192.168.203.107@tcp:38:kworker.0 00010000:00010000:0.0:1789764113.014947:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764113.270946:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789764114.039068:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764114.294965:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789764115.063011:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764115.318958:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789764116.087088:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764116.342939:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789764117.110971:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764117.366923:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:3.0:1789764117.878981:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764117.878992:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789764117.878996:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764117.879005:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 3551 00000100:00080000:3.0:1789764117.879014:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764117.879021:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:3.0:1789764117.879067:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872ea00 x1876700024799744/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764117.879096:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764117.879113:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789764117.879115:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764117.879120:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 3551 00000100:00080000:3.0:1789764117.879125:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764117.879129:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789764117.879138:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764117.879147:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024799744:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789764117.879154:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872dc00 x1876700024799872/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764117.879168:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764117.879171:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789764117.879173:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764117.879177:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 3551 00000100:00080000:3.0:1789764117.879180:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764117.879184:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789764117.879189:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764117.879199:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872e680 x1876700024800000/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789764117.879209:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789764117.879246:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764117/real 1789764117] req@ffffa06c1872ea00 x1876700024799744/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764142 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764117.879262:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872ea00 x1876700024799744/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764142 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764117.879284:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764117.879289:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789764117.879292:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024799744:192.168.203.107@tcp:250:kworker.0 00000100:00100000:0.0:1789764117.879313:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024799872:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789764117.879335:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789764117.879346:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764117/real 1789764117] req@ffffa06c1872dc00 x1876700024799872/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764142 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764117.879357:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872dc00 x1876700024799872/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764142 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764117.879365:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764117.879368:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789764117.879372:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024799872:192.168.203.107@tcp:38:kworker.0 00000100:00100000:0.0:1789764117.879377:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024800000:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789764117.879394:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789764117.879404:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764117/real 1789764117] req@ffffa06c1872e680 x1876700024800000/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764142 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764117.879414:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872e680 x1876700024800000/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764142 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764117.879421:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764117.879424:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789764117.879426:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024800000:192.168.203.107@tcp:38:kworker.0 00010000:00010000:2.0:1789764118.135165:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764118.390949:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789764119.158998:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764119.414928:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789764120.182978:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764120.438936:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789764121.206957:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764121.462956:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789764122.230990:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764122.486967:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789764122.998957:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764122.998967:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789764122.998970:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764122.998979:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 3556 00000100:00080000:0.0:1789764122.998999:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764122.999006:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789764122.999040:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0000 x1876700024800128/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764122.999060:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764122.999063:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789764122.999064:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764122.999067:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 3556 00000100:00080000:0.0:1789764122.999070:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764122.999072:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789764122.999079:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764122.999099:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d3b80 x1876700024800256/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789764122.999102:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024800128:192.168.203.107@tcp:250:kworker.0 00000100:00080000:0.0:1789764122.999113:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764122.999116:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789764122.999117:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764122.999119:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 3556 00000100:00080000:0.0:1789764122.999121:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764122.999123:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789764122.999138:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764122.999147:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d2300 x1876700024800384/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789764122.999171:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789764122.999212:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764122/real 1789764122] req@ffffa06c091d0000 x1876700024800128/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764147 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764122.999230:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0000 x1876700024800128/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764147 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764122.999239:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764122.999243:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789764122.999247:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024800128:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789764122.999278:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024800256:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789764122.999300:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789764122.999314:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764122/real 1789764122] req@ffffa06c091d3b80 x1876700024800256/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764147 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764122.999326:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d3b80 x1876700024800256/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764147 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764122.999334:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764122.999337:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789764122.999341:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024800256:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789764122.999347:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024800384:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789764122.999367:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789764122.999379:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764122/real 1789764122] req@ffffa06c091d2300 x1876700024800384/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764147 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764122.999405:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d2300 x1876700024800384/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764147 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764122.999414:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764122.999417:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789764122.999420:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024800384:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789764123.255015:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764123.510929:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789764124.279051:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764124.534953:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789764125.303004:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764125.558952:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789764126.326988:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764126.582940:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789764127.351045:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764127.606986:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789764128.118925:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764128.118936:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789764128.118940:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764128.118949:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 3561 00000100:00080000:0.0:1789764128.118971:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764128.118978:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789764128.119016:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d2300 x1876700024800512/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764128.119051:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764128.119055:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789764128.119057:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764128.119062:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 3561 00000100:00080000:0.0:1789764128.119065:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764128.119069:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789764128.119077:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764128.119089:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d3b80 x1876700024800640/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764128.119100:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764128.119104:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789764128.119105:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764128.119108:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 3561 00000100:00080000:0.0:1789764128.119126:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764128.119130:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789764128.119136:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764128.119146:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0000 x1876700024800768/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789764128.119239:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024800512:192.168.203.107@tcp:250:kworker.0 00000800:00000100:3.0:1789764128.119306:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789764128.119355:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764128/real 1789764128] req@ffffa06c091d2300 x1876700024800512/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764153 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764128.119372:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d2300 x1876700024800512/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764153 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764128.119382:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764128.119386:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789764128.119390:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024800512:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789764128.119398:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024800640:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789764128.119428:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789764128.119440:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764128/real 1789764128] req@ffffa06c091d3b80 x1876700024800640/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764153 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764128.119452:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d3b80 x1876700024800640/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764153 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764128.119461:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764128.119464:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789764128.119467:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024800640:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789764128.119473:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024800768:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789764128.119490:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789764128.119502:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764128/real 1789764128] req@ffffa06c091d0000 x1876700024800768/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764153 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764128.119512:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0000 x1876700024800768/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764153 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764128.119519:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764128.119523:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789764128.119526:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024800768:192.168.203.107@tcp:38:kworker.0 00010000:00010000:2.0:1789764128.375641:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764128.630971:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789764129.398967:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764129.654977:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789764130.422986:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764130.678973:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789764131.447055:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764131.702963:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789764132.471014:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764132.726952:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:1.0:1789764133.238991:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764133.238999:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789764133.239002:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764133.239009:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 3567 00000100:00080000:1.0:1789764133.239017:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764133.239023:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:1.0:1789764133.239050:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2d500 x1876700024800896/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764133.239070:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764133.239073:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789764133.239074:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764133.239076:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 3567 00000100:00080000:1.0:1789764133.239078:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764133.239080:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789764133.239086:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764133.239096:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2ca80 x1876700024801024/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764133.239103:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764133.239106:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789764133.239107:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764133.239111:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 3567 00000100:00100000:3.0:1789764133.239113:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024800896:192.168.203.107@tcp:250:kworker.0 00000100:00080000:1.0:1789764133.239124:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764133.239128:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:1.0:1789764133.239132:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764133.239139:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c08e2e300 x1876700024801152/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789764133.239173:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789764133.239205:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764133/real 1789764133] req@ffffa06c08e2d500 x1876700024800896/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764158 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764133.239217:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2d500 x1876700024800896/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764158 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764133.239224:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764133.239226:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789764133.239229:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024800896:192.168.203.107@tcp:250:kworker.0 00000100:00100000:3.0:1789764133.239234:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024801024:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789764133.239248:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789764133.239255:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764133/real 1789764133] req@ffffa06c08e2ca80 x1876700024801024/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764158 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764133.239262:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2ca80 x1876700024801024/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764158 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764133.239267:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764133.239269:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789764133.239271:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024801024:192.168.203.107@tcp:38:kworker.0 00000100:00100000:3.0:1789764133.239288:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c08e2e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024801152:192.168.203.107@tcp:38:kworker.0 00000800:00000100:3.0:1789764133.239297:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:3.0:1789764133.239303:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764133/real 1789764133] req@ffffa06c08e2e300 x1876700024801152/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764158 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764133.239317:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c08e2e300 x1876700024801152/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764158 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764133.239321:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764133.239323:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:3.0:1789764133.239325:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c08e2e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024801152:192.168.203.107@tcp:38:kworker.0 00010000:00010000:1.0:1789764133.494989:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764133.750961:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789764134.518967:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764134.774941:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789764135.543109:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764135.798975:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789764136.566981:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764136.822908:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789764137.591031:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764137.846971:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:3.0:1789764138.358981:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764138.358990:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789764138.358994:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764138.359011:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 3572 00000100:00080000:3.0:1789764138.359022:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764138.359029:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:3.0:1789764138.359066:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872c700 x1876700024801280/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764138.359100:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764138.359105:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789764138.359116:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764138.359122:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 3572 00000100:00080000:3.0:1789764138.359139:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764138.359144:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789764138.359153:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764138.359168:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872f100 x1876700024801408/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789764138.359173:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024801280:192.168.203.107@tcp:250:kworker.0 00000100:00080000:3.0:1789764138.359191:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764138.359195:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789764138.359197:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764138.359201:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 3572 00000100:00080000:3.0:1789764138.359204:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764138.359209:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789764138.359213:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764138.359233:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872ca80 x1876700024801536/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789764138.359236:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789764138.359275:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764138/real 1789764138] req@ffffa06c1872c700 x1876700024801280/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764163 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764138.359301:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872c700 x1876700024801280/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764163 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764138.359311:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764138.359315:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789764138.359319:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024801280:192.168.203.107@tcp:250:kworker.0 00000100:00100000:0.0:1789764138.359341:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024801408:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789764138.359366:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789764138.359375:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764138/real 1789764138] req@ffffa06c1872f100 x1876700024801408/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764163 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764138.359383:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872f100 x1876700024801408/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764163 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764138.359388:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764138.359396:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789764138.359398:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024801408:192.168.203.107@tcp:38:kworker.0 00000100:00100000:0.0:1789764138.359402:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024801536:192.168.203.107@tcp:38:kworker.0 00000800:00000100:0.0:1789764138.359414:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:0.0:1789764138.359420:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764138/real 1789764138] req@ffffa06c1872ca80 x1876700024801536/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764163 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764138.359427:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872ca80 x1876700024801536/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764163 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764138.359432:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764138.359434:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:0.0:1789764138.359435:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024801536:192.168.203.107@tcp:38:kworker.0 00010000:00010000:0.0:1789764138.615085:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764138.870949:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789764139.638987:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764139.894948:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789764140.663051:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764140.918946:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789764141.686972:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789764141.942972:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789764142.710995:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789764142.966952:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789764143.478999:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764143.479013:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789764143.479018:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764143.479033:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 3577 00000100:00080000:0.0:1789764143.479046:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764143.479055:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789764143.479108:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d1500 x1876700024801664/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764143.479157:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764143.479161:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789764143.479164:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764143.479173:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 3577 00000100:00080000:0.0:1789764143.479179:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764143.479184:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789764143.479196:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764143.479198:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024801664:192.168.203.107@tcp:250:kworker.0 00000100:00100000:0.0:1789764143.479233:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0e00 x1876700024801792/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764143.479248:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764143.479253:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789764143.479256:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764143.479260:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 3577 00000100:00080000:0.0:1789764143.479265:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764143.479270:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000800:00000100:1.0:1789764143.479278:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000008:00100000:0.0:1789764143.479282:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764143.479299:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d2a00 x1876700024801920/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789764143.479335:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764143/real 1789764143] req@ffffa06c091d1500 x1876700024801664/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764168 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764143.479358:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d1500 x1876700024801664/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764168 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764143.479370:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764143.479375:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789764143.479380:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024801664:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789764143.479407:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024801792:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789764143.479431:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789764143.479444:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764143/real 1789764143] req@ffffa06c091d0e00 x1876700024801792/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764168 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764143.479457:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0e00 x1876700024801792/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764168 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764143.479478:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764143.479482:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789764143.479486:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024801792:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789764143.479493:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024801920:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789764143.479511:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789764143.479525:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764143/real 1789764143] req@ffffa06c091d2a00 x1876700024801920/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764168 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764143.479538:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d2a00 x1876700024801920/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764168 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764143.479543:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764143.479546:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789764143.479548:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024801920:192.168.203.107@tcp:38:kworker.0 00010000:00010000:0.0:1789764143.734983:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789764143.990969:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789764144.759176:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789764145.014943:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789764145.783082:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789764146.038954:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789764146.806993:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789764147.062911:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789764147.831903:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789764148.086966:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:2.0:1789764148.599034:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764148.599042:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789764148.599046:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764148.599055:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 3582 00000100:00080000:2.0:1789764148.599064:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764148.599070:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:2.0:1789764148.599107:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51880 x1876700024802048/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764148.599520:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764148.599524:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789764148.599538:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764148.599542:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 3582 00000100:00080000:2.0:1789764148.599547:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764148.599551:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789764148.599559:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764148.599571:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e52300 x1876700024802176/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764148.599582:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764148.599585:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789764148.599587:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764148.599590:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 3582 00000100:00080000:2.0:1789764148.599593:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764148.599596:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:2.0:1789764148.599600:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764148.599609:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c07e51180 x1876700024802304/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789764148.599879:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024802048:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789764148.599960:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e52300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024802176:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789764148.599987:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c07e51180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024802304:192.168.203.107@tcp:38:kworker.0 00010000:00010000:1.0:1789764148.855082:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789764149.110943:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789764149.880107:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764150.135053:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:2.0:1789764150.903097:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764151.158954:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000400:00000100:0.0:1789764151.673127:0:2328:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.107:988 00000400:00000100:0.0:1789764151.673170:0:2328:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.107@tcp at host 192.168.203.107 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1789764151.673185:0:2328:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.107@tcp 00000800:00000100:0.0:1789764151.673227:0:2328:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.107@tcp 00000800:00000100:0.0:1789764151.673236:0:2328:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.107@tcp 00000100:00100000:1.0:1789764151.674923:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764148/real 1789764151] req@ffffa06c07e51880 x1876700024802048/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764173 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764151.674981:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51880 x1876700024802048/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764173 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764151.674993:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764151.674997:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789764151.675001:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024802048:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789764151.675009:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764148/real 1789764151] req@ffffa06c07e52300 x1876700024802176/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764173 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764151.675022:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e52300 x1876700024802176/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764173 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764151.675032:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764151.675035:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789764151.675038:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e52300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024802176:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789764151.675044:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764148/real 1789764151] req@ffffa06c07e51180 x1876700024802304/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764173 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764151.675055:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c07e51180 x1876700024802304/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764173 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764151.675063:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764151.675066:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789764151.675069:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c07e51180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024802304:192.168.203.107@tcp:38:kworker.0 00010000:00010000:1.0:1789764151.927040:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764152.182966:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789764152.951093:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764153.207016:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:0.0:1789764153.719830:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764153.719840:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789764153.719862:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764153.719872:0:10041:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 3587 00000100:00080000:0.0:1789764153.719882:0:10041:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764153.719889:0:10041:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:0.0:1789764153.719921:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d2680 x1876700024802432/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764153.719952:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764153.719955:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789764153.719957:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764153.719961:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 3587 00000100:00080000:0.0:1789764153.719965:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764153.719968:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789764153.719976:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764153.719986:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d0380 x1876700024802560/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764153.720011:0:10041:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764153.720014:0:10041:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789764153.720016:0:10041:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764153.720020:0:10041:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 3587 00000100:00080000:0.0:1789764153.720023:0:10041:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764153.720027:0:10041:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:0.0:1789764153.720032:0:10041:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764153.720042:0:10041:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c091d3100 x1876700024802688/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789764153.720139:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024802432:192.168.203.107@tcp:250:kworker.0 00000800:00000100:1.0:1789764153.720193:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789764153.720282:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764153/real 1789764153] req@ffffa06c091d2680 x1876700024802432/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764178 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764153.720299:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d2680 x1876700024802432/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764178 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764153.720309:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764153.720313:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789764153.720317:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024802432:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789764153.720324:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024802560:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789764153.720345:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789764153.720358:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764153/real 1789764153] req@ffffa06c091d0380 x1876700024802560/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764178 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764153.720369:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d0380 x1876700024802560/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764178 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764153.720376:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764153.720379:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789764153.720382:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024802560:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789764153.720387:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c091d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024802688:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789764153.720406:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789764153.720415:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764153/real 1789764153] req@ffffa06c091d3100 x1876700024802688/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764178 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764153.720565:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c091d3100 x1876700024802688/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764178 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764153.720573:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764153.720577:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789764153.720580:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c091d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024802688:192.168.203.107@tcp:38:kworker.0 00010000:00010000:2.0:1789764153.974968:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764154.231947:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789764154.999010:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764155.255002:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:3.0:1789764156.023048:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764156.278947:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:0.0:1789764157.047110:0:5427:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764157.302944:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00010000:00010000:1.0:1789764158.071695:0:5428:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764158.326948:0:9931:0:(client.c:2715:ptlrpc_set_wait()) set ffffa06c200eb600 going to sleep for 0 seconds 00000100:00080000:3.0:1789764158.839697:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 02bc4b43-5e6f-4d7e-9224-80f5c789104f->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764158.839707:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789764158.839711:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764158.839721:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.107@tcp: connect to NID 192.168.203.107@tcp last attempt 3592 00000100:00080000:3.0:1789764158.839745:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764158.839754:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.107@tcp: import ffffa06c108b6000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000100:00100000:3.0:1789764158.839799:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872d180 x1876700024802816/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764158.839829:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 734aaee5-e21a-40f9-9d4e-90e3c76738d2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764158.839833:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789764158.839834:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764158.839839:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: connect to NID 192.168.203.107@tcp last attempt 3592 00000100:00080000:3.0:1789764158.839842:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764158.840023:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c18a1e000: import ffffa06c07e43000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789764158.840033:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764158.840047:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872e300 x1876700024802944/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764158.840058:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) b837c2fb-d1c1-4622-807e-a243f6fc2450->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764158.840061:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789764158.840063:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764158.840066:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: connect to NID 192.168.203.107@tcp last attempt 3592 00000100:00080000:3.0:1789764158.840083:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764158.840087:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffffa06c07e46000: import ffffa06c09137000 using connection 192.168.203.107@tcp/192.168.203.107@tcp 00000008:00100000:3.0:1789764158.840092:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764158.840102:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06c1872ce00 x1876700024803072/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789764158.840694:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024802816:192.168.203.107@tcp:250:kworker.0 00000800:00000100:1.0:1789764158.840759:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789764158.840803:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764158/real 1789764158] req@ffffa06c1872d180 x1876700024802816/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764183 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764158.840821:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872d180 x1876700024802816/t0(0) o250->MGC192.168.203.107@tcp@192.168.203.107@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764183 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764158.840830:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c108b6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764158.840834:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789764158.840838:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:02bc4b43-5e6f-4d7e-9224-80f5c789104f:2340:1876700024802816:192.168.203.107@tcp:250:kworker.0 00000100:00100000:1.0:1789764158.840865:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024802944:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789764158.840885:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789764158.840898:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764158/real 1789764158] req@ffffa06c1872e300 x1876700024802944/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764183 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764158.840911:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872e300 x1876700024802944/t0(0) o38->lustre-MDT0000-mdc-ffffa06c18a1e000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764183 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764158.840927:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c07e43000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764158.840931:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789764158.840934:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:734aaee5-e21a-40f9-9d4e-90e3c76738d2:2340:1876700024802944:192.168.203.107@tcp:38:kworker.0 00000100:00100000:1.0:1789764158.840940:0:2340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffffa06c1872ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024803072:192.168.203.107@tcp:38:kworker.0 00000800:00000100:1.0:1789764158.840960:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.107@tcp 00000100:00100000:1.0:1789764158.840971:0:2340:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764158/real 1789764158] req@ffffa06c1872ce00 x1876700024803072/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764183 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764158.840981:0:2340:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06c1872ce00 x1876700024803072/t0(0) o38->lustre-MDT0000-mdc-ffffa06c07e46000@192.168.203.107@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764183 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764158.840989:0:2340:0:(import.c:65:import_set_state_nolock()) ffffa06c09137000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764158.840991:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.107@tcp failed (-110) 00000100:00100000:1.0:1789764158.840994:0:2340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffffa06c1872ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:b837c2fb-d1c1-4622-807e-a243f6fc2450:2340:1876700024803072:192.168.203.107@tcp:38:kworker.0 00010000:00010000:3.0:1789764159.095051:0:9377:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) Debug log: 37181 lines, 37181 kept, 0 dropped, 0 bad.